diff --git a/yt/rust/.gitignore b/yt/rust/.gitignore new file mode 100644 index 000000000000..eccd7b4ab85e --- /dev/null +++ b/yt/rust/.gitignore @@ -0,0 +1,2 @@ +/target/ +**/*.rs.bk diff --git a/yt/rust/Cargo.lock b/yt/rust/Cargo.lock new file mode 100644 index 000000000000..ac4ea5bb1a3e --- /dev/null +++ b/yt/rust/Cargo.lock @@ -0,0 +1,2046 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "iri-string", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "ytsaurus-client" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "dirs", + "futures", + "pin-project-lite", + "reqwest", + "serde", + "thiserror", + "tokio", + "tracing", + "uuid", + "wiremock", + "ytsaurus-yson", +] + +[[package]] +name = "ytsaurus-yson" +version = "0.1.0" +dependencies = [ + "bytes", + "indexmap", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" diff --git a/yt/rust/Cargo.toml b/yt/rust/Cargo.toml new file mode 100644 index 000000000000..931d15781d4a --- /dev/null +++ b/yt/rust/Cargo.toml @@ -0,0 +1,22 @@ +[workspace] +resolver = "2" +members = [ + "ytsaurus-yson", + "ytsaurus-client", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/ytsaurus/ytsaurus" + +[workspace.dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "1.0" +bytes = "1.5" +tokio = { version = "1", features = ["full"] } +futures = "0.3" +async-trait = "0.1" +indexmap = { version = "2.0", features = ["serde"] } diff --git a/yt/rust/README.md b/yt/rust/README.md new file mode 100644 index 000000000000..a30c4456f887 --- /dev/null +++ b/yt/rust/README.md @@ -0,0 +1,197 @@ +# YTsaurus Rust Client + +A native Rust client for YTsaurus, a distributed storage and processing platform for big data. + +## Crates + +This workspace contains two crates: + +- **ytsaurus-yson** - YSON serialization format library with serde support +- **ytsaurus-client** - YTsaurus client library with Cypress and table operations + +## Quick Start + +Add to your `Cargo.toml`: + +```toml +[dependencies] +ytsaurus-client = { path = "path/to/yt/rust/ytsaurus-client" } +tokio = { version = "1", features = ["full"] } +serde = { version = "1.0", features = ["derive"] } +``` + +## Usage + +### Basic Cypress Operations + +```rust +use ytsaurus_client::{YtClient, YPath, NodeType, CreateNodeOptions, RemoveNodeOptions}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Create client from environment (YT_PROXY, YT_TOKEN) + let client = YtClient::from_env()?; + + // Or use builder pattern + let client = YtClient::builder() + .proxy("localhost:8000") + .token("your-token") + .build()?; + + // Create a directory + let path = YPath::new("//tmp/my_dir"); + client.create_node(&path, NodeType::Map, Some(CreateNodeOptions::recursive())).await?; + + // Check if path exists + let exists = client.exists(&path, None).await?; + println!("Path exists: {}", exists); + + // Set a value + client.set_node(&path.join("key"), &"hello", None).await?; + + // Get a value + let value: String = client.get_node(&path.join("key"), None).await?; + println!("Value: {}", value); + + // List directory contents + let children = client.list_node(&path, None).await?; + println!("Children: {:?}", children); + + // Remove the directory + client.remove_node(&path, Some(RemoveNodeOptions::recursive())).await?; + + Ok(()) +} +``` + +### Table Read/Write + +```rust +use serde::{Serialize, Deserialize}; +use ytsaurus_client::{YtClient, YPath}; + +#[derive(Serialize, Deserialize, Debug)] +struct User { + id: i64, + name: String, + email: String, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let client = YtClient::from_env()?; + let table_path = YPath::new("//tmp/users"); + + // Write rows to a table + let users = vec![ + User { id: 1, name: "Alice".into(), email: "alice@example.com".into() }, + User { id: 2, name: "Bob".into(), email: "bob@example.com".into() }, + ]; + + let mut writer = client.write_table(&table_path, None).await?; + for user in &users { + writer.write(user)?; + } + writer.commit().await?; + + // Read rows from a table + let reader = client.read_table::(&table_path, None).await?; + let read_users = reader.collect().await?; + for user in read_users { + println!("{:?}", user); + } + + Ok(()) +} +``` + +### YSON Serialization + +```rust +use ytsaurus_client::yson::{to_binary, from_binary, YsonValue}; +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +struct Config { + name: String, + count: i64, + enabled: bool, +} + +fn main() -> Result<(), Box> { + let config = Config { + name: "test".into(), + count: 42, + enabled: true, + }; + + // Serialize to binary YSON + let bytes = to_binary(&config)?; + + // Deserialize from binary YSON + let decoded: Config = from_binary(&bytes)?; + assert_eq!(config, decoded); + + Ok(()) +} +``` + +## Configuration + +The client can be configured via environment variables: + +- `YT_PROXY` - Proxy address (e.g., `localhost:8000`) +- `YT_TOKEN` - OAuth token for authentication +- `YT_TOKEN_FILE` - Path to file containing the token + +Or via the builder pattern: + +```rust +let client = YtClient::builder() + .proxy("cluster.example.com:8000") + .token("your-oauth-token") + .use_tls(true) + .request_timeout(Duration::from_secs(60)) + .build()?; +``` + +## API Reference + +### Cypress Operations + +| Method | Description | +|--------|-------------| +| `create_node` | Create a new node (map, table, file, etc.) | +| `remove_node` | Remove a node | +| `exists` | Check if a node exists | +| `get_node` | Get a node's value | +| `set_node` | Set a node's value | +| `list_node` | List children of a map node | +| `copy_node` | Copy a node | +| `move_node` | Move a node | +| `link_node` | Create a symbolic link | + +### Table Operations + +| Method | Description | +|--------|-------------| +| `read_table` | Read rows from a table | +| `write_table` | Write rows to a table | + +## Building + +```bash +cd yt/rust +cargo build +``` + +## Testing + +```bash +cd yt/rust +cargo test +``` + +## License + +Apache 2.0 diff --git a/yt/rust/examples/basic.rs b/yt/rust/examples/basic.rs new file mode 100644 index 000000000000..e513ea6b87fa --- /dev/null +++ b/yt/rust/examples/basic.rs @@ -0,0 +1,121 @@ +//! Basic usage example for YTsaurus Rust client. +//! +//! This example demonstrates: +//! - Creating a client from environment variables +//! - Basic Cypress operations (create, get, set, list, remove) +//! - Table read/write operations + +use serde::{Deserialize, Serialize}; +use ytsaurus_client::{YtClient, YPath, NodeType, CreateNodeOptions, RemoveNodeOptions}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct User { + id: i64, + name: String, + email: String, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Initialize tracing for logging + tracing_subscriber::fmt::init(); + + // Create client from environment variables + // Set YT_PROXY and optionally YT_TOKEN before running + let client = YtClient::builder() + .proxy_from_env() + .token_from_env_or_file() + .build()?; + + println!("Connected to YTsaurus at {}", client.config().proxy); + + // Example path for our test data + let base_path = YPath::new("//tmp/rust_client_example"); + + // Create a directory + println!("\n1. Creating directory..."); + client + .create_node( + &base_path, + NodeType::Map, + Some(CreateNodeOptions::recursive()), + ) + .await?; + println!(" Created: {}", base_path); + + // Check if it exists + println!("\n2. Checking existence..."); + let exists = client.exists(&base_path, None).await?; + println!(" Exists: {}", exists); + + // Set a value + println!("\n3. Setting a value..."); + let value_path = base_path.join("config"); + client + .set_node(&value_path, &"hello from rust", None) + .await?; + println!(" Set value at: {}", value_path); + + // Get the value back + println!("\n4. Getting value..."); + let value: String = client.get_node(&value_path, None).await?; + println!(" Value: {}", value); + + // List directory contents + println!("\n5. Listing directory..."); + let children = client.list_node(&base_path, None).await?; + println!(" Children: {:?}", children); + + // Write a table + println!("\n6. Writing table..."); + let table_path = base_path.join("users"); + + // Create users + let users = vec![ + User { + id: 1, + name: "Alice".to_string(), + email: "alice@example.com".to_string(), + }, + User { + id: 2, + name: "Bob".to_string(), + email: "bob@example.com".to_string(), + }, + User { + id: 3, + name: "Charlie".to_string(), + email: "charlie@example.com".to_string(), + }, + ]; + + let mut writer = client.write_table(&table_path, None).await?; + for user in &users { + writer.write(user)?; + } + writer.commit().await?; + println!(" Wrote {} users to: {}", users.len(), table_path); + + // Read the table back + println!("\n7. Reading table..."); + let reader = client.read_table::(&table_path, None).await?; + let read_users = reader.collect().await?; + println!(" Read {} users:", read_users.len()); + for user in &read_users { + println!(" - {:?}", user); + } + + // Verify data matches + assert_eq!(users, read_users); + println!("\n Data verified successfully!"); + + // Clean up + println!("\n8. Cleaning up..."); + client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await?; + println!(" Removed: {}", base_path); + + println!("\nExample completed successfully!"); + Ok(()) +} diff --git a/yt/rust/ytsaurus-client/Cargo.toml b/yt/rust/ytsaurus-client/Cargo.toml new file mode 100644 index 000000000000..6e7530062d7e --- /dev/null +++ b/yt/rust/ytsaurus-client/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "ytsaurus-client" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Rust client for YTsaurus distributed storage and processing platform" +keywords = ["ytsaurus", "distributed", "storage", "bigdata"] +categories = ["database", "api-bindings"] + +[dependencies] +ytsaurus-yson = { path = "../ytsaurus-yson" } +serde.workspace = true +thiserror.workspace = true +tokio.workspace = true +futures.workspace = true +async-trait.workspace = true +reqwest = { version = "0.12", features = ["json", "stream", "gzip"] } +uuid = { version = "1.6", features = ["v4"] } +tracing = "0.1" +dirs = "5.0" +pin-project-lite = "0.2" +bytes = "1.5" + +[dev-dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +wiremock = "0.6" diff --git a/yt/rust/ytsaurus-client/src/auth.rs b/yt/rust/ytsaurus-client/src/auth.rs new file mode 100644 index 000000000000..c39bc7101b6a --- /dev/null +++ b/yt/rust/ytsaurus-client/src/auth.rs @@ -0,0 +1,144 @@ +//! Authentication and credentials. + +use reqwest::header::{HeaderMap, HeaderValue}; + +/// Trait for authentication credentials +pub trait Credentials: Send + Sync { + /// Apply credentials to request headers + fn apply_headers(&self, headers: &mut HeaderMap); +} + +/// OAuth token credentials +#[derive(Debug, Clone)] +pub struct TokenCredentials { + /// OAuth token + pub token: String, +} + +impl TokenCredentials { + /// Create new token credentials + pub fn new(token: impl Into) -> Self { + Self { + token: token.into(), + } + } +} + +impl Credentials for TokenCredentials { + fn apply_headers(&self, headers: &mut HeaderMap) { + if let Ok(value) = HeaderValue::from_str(&format!("OAuth {}", self.token)) { + headers.insert("Authorization", value); + } + } +} + +/// Bearer token credentials +#[derive(Debug, Clone)] +pub struct BearerCredentials { + /// Bearer token + pub token: String, +} + +impl BearerCredentials { + /// Create new bearer credentials + pub fn new(token: impl Into) -> Self { + Self { + token: token.into(), + } + } +} + +impl Credentials for BearerCredentials { + fn apply_headers(&self, headers: &mut HeaderMap) { + if let Ok(value) = HeaderValue::from_str(&format!("Bearer {}", self.token)) { + headers.insert("Authorization", value); + } + } +} + +/// Service ticket credentials (TVM) +#[derive(Debug, Clone)] +pub struct ServiceTicketCredentials { + /// Service ticket + pub ticket: String, +} + +impl ServiceTicketCredentials { + /// Create new service ticket credentials + pub fn new(ticket: impl Into) -> Self { + Self { + ticket: ticket.into(), + } + } +} + +impl Credentials for ServiceTicketCredentials { + fn apply_headers(&self, headers: &mut HeaderMap) { + if let Ok(value) = HeaderValue::from_str(&self.ticket) { + headers.insert("X-Ya-Service-Ticket", value); + } + } +} + +/// User ticket credentials (TVM) +#[derive(Debug, Clone)] +pub struct UserTicketCredentials { + /// User ticket + pub ticket: String, +} + +impl UserTicketCredentials { + /// Create new user ticket credentials + pub fn new(ticket: impl Into) -> Self { + Self { + ticket: ticket.into(), + } + } +} + +impl Credentials for UserTicketCredentials { + fn apply_headers(&self, headers: &mut HeaderMap) { + if let Ok(value) = HeaderValue::from_str(&self.ticket) { + headers.insert("X-Ya-User-Ticket", value); + } + } +} + +/// No credentials (anonymous) +#[derive(Debug, Clone, Default)] +pub struct NoCredentials; + +impl Credentials for NoCredentials { + fn apply_headers(&self, _headers: &mut HeaderMap) { + // No-op + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_token_credentials() { + let creds = TokenCredentials::new("my-token"); + let mut headers = HeaderMap::new(); + creds.apply_headers(&mut headers); + + assert_eq!( + headers.get("Authorization").unwrap().to_str().unwrap(), + "OAuth my-token" + ); + } + + #[test] + fn test_bearer_credentials() { + let creds = BearerCredentials::new("my-token"); + let mut headers = HeaderMap::new(); + creds.apply_headers(&mut headers); + + assert_eq!( + headers.get("Authorization").unwrap().to_str().unwrap(), + "Bearer my-token" + ); + } +} diff --git a/yt/rust/ytsaurus-client/src/client.rs b/yt/rust/ytsaurus-client/src/client.rs new file mode 100644 index 000000000000..f65932d46012 --- /dev/null +++ b/yt/rust/ytsaurus-client/src/client.rs @@ -0,0 +1,550 @@ +//! YTsaurus client implementation. + +use crate::auth::{Credentials, NoCredentials, TokenCredentials}; +use crate::config::{ClientBuilder, ClientConfig}; +use crate::cypress::*; +use crate::error::{ErrorCode, Result, YtError}; +use crate::path::YPath; +use crate::table::{ReadTableOptions, TableReader, TableWriter, WriteTableOptions}; +use reqwest::header::{HeaderMap, HeaderValue}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::sync::Arc; +use uuid::Uuid; +use ytsaurus_yson::{from_binary, to_binary}; + +/// Response wrapper - YT API returns responses in {value=...} format +#[derive(serde::Deserialize)] +struct ApiResponse { + value: T, +} + +/// Response wrapper for create commands - returns {node_id=...} +#[derive(serde::Deserialize)] +struct CreateResponse { + node_id: String, +} + +/// YTsaurus client +pub struct YtClient { + config: ClientConfig, + pub(crate) http_client: reqwest::Client, + pub(crate) credentials: Arc, +} + +impl YtClient { + /// Create a new client with the given configuration + pub fn new(config: ClientConfig) -> Result { + let http_client = reqwest::Client::builder() + .timeout(config.request_timeout) + .build() + .map_err(|e| YtError::new(ErrorCode::TransportError, e.to_string()))?; + + let credentials: Arc = match &config.token { + Some(token) if !token.is_empty() => Arc::new(TokenCredentials::new(token.clone())), + _ => Arc::new(NoCredentials), + }; + + Ok(Self { + config, + http_client, + credentials, + }) + } + + /// Create a client builder + pub fn builder() -> ClientBuilder { + ClientBuilder::new() + } + + /// Create a client from environment variables + pub fn from_env() -> Result { + ClientBuilder::new() + .proxy_from_env() + .token_from_env_or_file() + .build() + } + + /// Get the configuration + pub fn config(&self) -> &ClientConfig { + &self.config + } + + /// Execute an API command + async fn execute( + &self, + command: &str, + params: &P, + ) -> Result { + let url = format!("{}/api/v4/{}", self.config.base_url(), command); + let correlation_id = Uuid::new_v4().to_string(); + + let params_yson = to_binary(params)?; + + let mut headers = HeaderMap::new(); + headers.insert( + "X-YT-Correlation-Id", + HeaderValue::from_str(&correlation_id).unwrap(), + ); + headers.insert( + "X-YT-Input-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "X-YT-Output-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "Content-Type", + HeaderValue::from_static("application/x-yson"), + ); + + // Apply authentication + self.credentials.apply_headers(&mut headers); + + // Apply impersonation if set + if let Some(ref user) = self.config.impersonation_user { + if let Ok(value) = HeaderValue::from_str(user) { + headers.insert("X-YT-User-Name", value); + } + } + + tracing::debug!(command = command, correlation_id = %correlation_id, "executing YT command"); + + let response = self + .http_client + .post(&url) + .headers(headers) + .body(params_yson) + .send() + .await?; + + let status = response.status(); + let response_bytes = response.bytes().await?; + + if !status.is_success() { + // Try to parse error from response + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new( + ErrorCode::Generic, + format!("HTTP error: {} - {:?}", status, String::from_utf8_lossy(&response_bytes)), + )); + } + + // YT API wraps responses in {value=...} + let response: ApiResponse = from_binary(&response_bytes).map_err(|e| { + YtError::new( + ErrorCode::ProtocolError, + format!("failed to parse response: {}", e), + ) + })?; + Ok(response.value) + } + + /// Execute an API command with direct response parsing (no {value=...} wrapper) + async fn execute_raw_parsed( + &self, + command: &str, + params: &P, + ) -> Result { + let url = format!("{}/api/v4/{}", self.config.base_url(), command); + let correlation_id = Uuid::new_v4().to_string(); + + let params_yson = to_binary(params)?; + + let mut headers = HeaderMap::new(); + headers.insert( + "X-YT-Correlation-Id", + HeaderValue::from_str(&correlation_id).unwrap(), + ); + headers.insert( + "X-YT-Input-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "X-YT-Output-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "Content-Type", + HeaderValue::from_static("application/x-yson"), + ); + + self.credentials.apply_headers(&mut headers); + + if let Some(ref user) = self.config.impersonation_user { + if let Ok(value) = HeaderValue::from_str(user) { + headers.insert("X-YT-User-Name", value); + } + } + + let response = self + .http_client + .post(&url) + .headers(headers) + .body(params_yson) + .send() + .await?; + + let status = response.status(); + let response_bytes = response.bytes().await?; + + if !status.is_success() { + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new( + ErrorCode::Generic, + format!("HTTP error: {} - {:?}", status, String::from_utf8_lossy(&response_bytes)), + )); + } + + // Parse directly without ApiResponse wrapper + let response: R = from_binary(&response_bytes).map_err(|e| { + YtError::new( + ErrorCode::ProtocolError, + format!("failed to parse response: {}", e), + ) + })?; + Ok(response) + } + + /// Execute an API command with raw YSON response + async fn execute_raw(&self, command: &str, params: &P) -> Result> { + let url = format!("{}/api/v4/{}", self.config.base_url(), command); + let correlation_id = Uuid::new_v4().to_string(); + + let params_yson = to_binary(params)?; + + let mut headers = HeaderMap::new(); + headers.insert( + "X-YT-Correlation-Id", + HeaderValue::from_str(&correlation_id).unwrap(), + ); + headers.insert( + "X-YT-Input-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "X-YT-Output-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "Content-Type", + HeaderValue::from_static("application/x-yson"), + ); + + self.credentials.apply_headers(&mut headers); + + if let Some(ref user) = self.config.impersonation_user { + if let Ok(value) = HeaderValue::from_str(user) { + headers.insert("X-YT-User-Name", value); + } + } + + let response = self + .http_client + .post(&url) + .headers(headers) + .body(params_yson) + .send() + .await?; + + let status = response.status(); + let response_bytes = response.bytes().await?; + + if !status.is_success() { + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new( + ErrorCode::Generic, + format!("HTTP error: {}", status), + )); + } + + Ok(response_bytes.to_vec()) + } + + // ==================== Cypress Operations ==================== + + /// Create a node in Cypress + pub async fn create_node( + &self, + path: &YPath, + node_type: NodeType, + options: Option, + ) -> Result { + let options = options.unwrap_or_default(); + let params = CreateNodeParams { + path: path.as_str(), + r#type: node_type.as_str(), + recursive: options.recursive, + ignore_existing: options.ignore_existing, + force: options.force, + attributes: options.attributes, + }; + let response: CreateResponse = self.execute_raw_parsed("create", ¶ms).await?; + Ok(response.node_id) + } + + /// Remove a node from Cypress + pub async fn remove_node( + &self, + path: &YPath, + options: Option, + ) -> Result<()> { + let options = options.unwrap_or_default(); + let params = RemoveNodeParams { + path: path.as_str(), + recursive: options.recursive, + force: options.force, + }; + self.execute_void("remove", ¶ms).await + } + + /// Execute an API command with no return value + async fn execute_void(&self, command: &str, params: &P) -> Result<()> { + let url = format!("{}/api/v4/{}", self.config.base_url(), command); + let correlation_id = Uuid::new_v4().to_string(); + + let params_yson = to_binary(params)?; + + let mut headers = HeaderMap::new(); + headers.insert( + "X-YT-Correlation-Id", + HeaderValue::from_str(&correlation_id).unwrap(), + ); + headers.insert("X-YT-Input-Format", HeaderValue::from_static("\"yson\"")); + headers.insert("X-YT-Output-Format", HeaderValue::from_static("\"yson\"")); + headers.insert("Content-Type", HeaderValue::from_static("application/x-yson")); + + self.credentials.apply_headers(&mut headers); + + if let Some(ref user) = self.config.impersonation_user { + if let Ok(value) = HeaderValue::from_str(user) { + headers.insert("X-YT-User-Name", value); + } + } + + let response = self + .http_client + .post(&url) + .headers(headers) + .body(params_yson) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let response_bytes = response.bytes().await?; + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new( + ErrorCode::Generic, + format!("HTTP error: {}", status), + )); + } + + Ok(()) + } + + /// Check if a node exists + pub async fn exists(&self, path: &YPath, options: Option) -> Result { + let params = ExistsParams { + path: path.as_str(), + }; + self.execute("exists", ¶ms).await + } + + /// Get a node value + pub async fn get_node( + &self, + path: &YPath, + options: Option, + ) -> Result { + let options = options.unwrap_or_default(); + let params = GetNodeParams { + path: path.as_str(), + attributes: options.attributes, + max_size: options.max_size, + }; + self.execute("get", ¶ms).await + } + + /// Set a node value + pub async fn set_node( + &self, + path: &YPath, + value: &T, + options: Option, + ) -> Result<()> { + let options = options.unwrap_or_default(); + let value_yson = to_binary(value)?; + + // Build parameters as JSON for header (X-YT-Parameters expects JSON) + let mut params_json = format!("{{\"path\":\"{}\"}}", path.as_str().replace('"', "\\\"")); + // Add optional parameters if set + if options.recursive || options.force { + params_json = format!("{{\"path\":\"{}\",\"recursive\":{},\"force\":{}}}", + path.as_str().replace('"', "\\\""), + options.recursive, + options.force + ); + } + + let url = format!("{}/api/v4/set", self.config.base_url()); + let correlation_id = Uuid::new_v4().to_string(); + + let mut headers = HeaderMap::new(); + headers.insert("X-YT-Correlation-Id", HeaderValue::from_str(&correlation_id).unwrap()); + headers.insert("X-YT-Input-Format", HeaderValue::from_static("\"yson\"")); + headers.insert("X-YT-Output-Format", HeaderValue::from_static("\"yson\"")); + headers.insert("X-YT-Parameters", HeaderValue::from_str(¶ms_json).unwrap()); + headers.insert("Content-Type", HeaderValue::from_static("application/x-yson")); + + self.credentials.apply_headers(&mut headers); + + let response = self + .http_client + .put(&url) + .headers(headers) + .body(value_yson) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let response_bytes = response.bytes().await?; + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new(ErrorCode::Generic, format!("HTTP error: {}", status))); + } + + Ok(()) + } + + /// List children of a node + pub async fn list_node( + &self, + path: &YPath, + options: Option, + ) -> Result> { + let options = options.unwrap_or_default(); + let params = ListNodeParams { + path: path.as_str(), + attributes: options.attributes, + max_size: options.max_size, + }; + self.execute("list", ¶ms).await + } + + /// Copy a node + pub async fn copy_node( + &self, + source: &YPath, + destination: &YPath, + options: Option, + ) -> Result { + let options = options.unwrap_or_default(); + let params = CopyNodeParams { + source_path: source.as_str(), + destination_path: destination.as_str(), + recursive: options.recursive, + force: options.force, + preserve_account: options.preserve_account, + preserve_expiration_time: options.preserve_expiration_time, + }; + let response: CreateResponse = self.execute_raw_parsed("copy", ¶ms).await?; + Ok(response.node_id) + } + + /// Move a node + pub async fn move_node( + &self, + source: &YPath, + destination: &YPath, + options: Option, + ) -> Result { + let options = options.unwrap_or_default(); + let params = MoveNodeParams { + source_path: source.as_str(), + destination_path: destination.as_str(), + recursive: options.recursive, + force: options.force, + preserve_account: options.preserve_account, + preserve_expiration_time: options.preserve_expiration_time, + }; + let response: CreateResponse = self.execute_raw_parsed("move", ¶ms).await?; + Ok(response.node_id) + } + + /// Create a symbolic link + pub async fn link_node( + &self, + target: &YPath, + link: &YPath, + options: Option, + ) -> Result { + let options = options.unwrap_or_default(); + let params = LinkNodeParams { + target_path: target.as_str(), + link_path: link.as_str(), + recursive: options.recursive, + force: options.force, + ignore_existing: options.ignore_existing, + }; + let response: CreateResponse = self.execute_raw_parsed("link", ¶ms).await?; + Ok(response.node_id) + } + + // ==================== Table Operations ==================== + + /// Read a table + pub async fn read_table( + &self, + path: &YPath, + options: Option, + ) -> Result> { + TableReader::new(self, path, options).await + } + + /// Write to a table + pub async fn write_table( + &self, + path: &YPath, + options: Option, + ) -> Result> { + TableWriter::new(self, path, options).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_builder() { + let client = YtClient::builder() + .proxy("localhost:8000") + .token("test-token") + .build() + .unwrap(); + + assert_eq!(client.config().proxy, "localhost:8000"); + } +} diff --git a/yt/rust/ytsaurus-client/src/config.rs b/yt/rust/ytsaurus-client/src/config.rs new file mode 100644 index 000000000000..60cd2c52dabd --- /dev/null +++ b/yt/rust/ytsaurus-client/src/config.rs @@ -0,0 +1,190 @@ +//! Client configuration. + +use crate::error::{Result, YtError, ErrorCode}; +use crate::YtClient; +use std::time::Duration; + +/// Client configuration +#[derive(Debug, Clone)] +pub struct ClientConfig { + /// Proxy address (e.g., "localhost:8000" or "cluster-name.yt.yandex.net") + pub proxy: String, + /// Use TLS + pub use_tls: bool, + /// OAuth or API token + pub token: Option, + /// Request timeout + pub request_timeout: Duration, + /// Impersonation user + pub impersonation_user: Option, +} + +impl ClientConfig { + /// Create default config with proxy + pub fn new(proxy: impl Into) -> Self { + Self { + proxy: proxy.into(), + use_tls: false, + token: None, + request_timeout: Duration::from_secs(300), + impersonation_user: None, + } + } + + /// Get the base URL for API requests + pub fn base_url(&self) -> String { + let scheme = if self.use_tls { "https" } else { "http" }; + format!("{}://{}", scheme, self.proxy) + } +} + +impl Default for ClientConfig { + fn default() -> Self { + Self { + proxy: String::new(), + use_tls: false, + token: None, + request_timeout: Duration::from_secs(300), + impersonation_user: None, + } + } +} + +/// Builder for creating YtClient +#[derive(Default)] +pub struct ClientBuilder { + config: ClientConfig, +} + +impl ClientBuilder { + /// Create a new builder + pub fn new() -> Self { + Self::default() + } + + /// Set the proxy address + pub fn proxy(mut self, proxy: impl Into) -> Self { + self.config.proxy = proxy.into(); + self + } + + /// Read proxy from YT_PROXY environment variable + pub fn proxy_from_env(mut self) -> Self { + if let Ok(proxy) = std::env::var("YT_PROXY") { + self.config.proxy = proxy; + } + self + } + + /// Set the OAuth token + pub fn token(mut self, token: impl Into) -> Self { + self.config.token = Some(token.into()); + self + } + + /// Read token from YT_TOKEN environment variable + pub fn token_from_env(mut self) -> Self { + if let Ok(token) = std::env::var("YT_TOKEN") { + self.config.token = Some(token); + } + self + } + + /// Read token from ~/.yt/token file + pub fn token_from_file(mut self) -> Self { + if let Some(token) = load_token_from_file() { + self.config.token = Some(token); + } + self + } + + /// Read token from YT_TOKEN env or ~/.yt/token file + pub fn token_from_env_or_file(self) -> Self { + let builder = self.token_from_env(); + if builder.config.token.is_some() { + return builder; + } + builder.token_from_file() + } + + /// Enable TLS + pub fn use_tls(mut self, use_tls: bool) -> Self { + self.config.use_tls = use_tls; + self + } + + /// Set request timeout + pub fn request_timeout(mut self, timeout: Duration) -> Self { + self.config.request_timeout = timeout; + self + } + + /// Set impersonation user + pub fn impersonation_user(mut self, user: impl Into) -> Self { + self.config.impersonation_user = Some(user.into()); + self + } + + /// Build the client + pub fn build(self) -> Result { + if self.config.proxy.is_empty() { + return Err(YtError::new( + ErrorCode::Generic, + "proxy address is required", + )); + } + YtClient::new(self.config) + } +} + +/// Load token from ~/.yt/token file +fn load_token_from_file() -> Option { + // Check YT_TOKEN_FILE first + if let Ok(token_file) = std::env::var("YT_TOKEN_FILE") { + if let Ok(token) = std::fs::read_to_string(&token_file) { + return Some(token.trim().to_string()); + } + } + + // Check ~/.yt/token + if let Some(home) = dirs::home_dir() { + let token_path = home.join(".yt").join("token"); + if let Ok(token) = std::fs::read_to_string(&token_path) { + return Some(token.trim().to_string()); + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_base_url() { + let config = ClientConfig::new("localhost:8000"); + assert_eq!(config.base_url(), "http://localhost:8000"); + + let mut config = ClientConfig::new("cluster.yt.example.com"); + config.use_tls = true; + assert_eq!(config.base_url(), "https://cluster.yt.example.com"); + } + + #[test] + fn test_builder() { + let result = ClientBuilder::new() + .proxy("localhost:8000") + .token("test-token") + .use_tls(false) + .build(); + + assert!(result.is_ok()); + } + + #[test] + fn test_builder_no_proxy() { + let result = ClientBuilder::new().build(); + assert!(result.is_err()); + } +} diff --git a/yt/rust/ytsaurus-client/src/cypress.rs b/yt/rust/ytsaurus-client/src/cypress.rs new file mode 100644 index 000000000000..e1513464e3b3 --- /dev/null +++ b/yt/rust/ytsaurus-client/src/cypress.rs @@ -0,0 +1,270 @@ +//! Cypress tree operations. + +use serde::Serialize; +use std::collections::HashMap; +use ytsaurus_yson::YsonValue; + +/// Node type in Cypress +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeType { + /// Map node (directory-like) + Map, + /// List node + List, + /// String value + String, + /// Int64 value + Int64, + /// Uint64 value + Uint64, + /// Double value + Double, + /// Boolean value + Boolean, + /// File node + File, + /// Table node + Table, + /// Document node (YSON document) + Document, + /// Symbolic link + Link, +} + +impl NodeType { + /// Get the type name as used in the API + pub fn as_str(&self) -> &'static str { + match self { + NodeType::Map => "map_node", + NodeType::List => "list_node", + NodeType::String => "string_node", + NodeType::Int64 => "int64_node", + NodeType::Uint64 => "uint64_node", + NodeType::Double => "double_node", + NodeType::Boolean => "boolean_node", + NodeType::File => "file", + NodeType::Table => "table", + NodeType::Document => "document", + NodeType::Link => "link", + } + } +} + +/// Node ID (GUID) +pub type NodeId = String; + +/// Options for create_node +#[derive(Debug, Clone, Default)] +pub struct CreateNodeOptions { + /// Create parent directories if needed + pub recursive: bool, + /// Don't fail if node already exists + pub ignore_existing: bool, + /// Overwrite existing node + pub force: bool, + /// Node attributes + pub attributes: Option>, +} + +impl CreateNodeOptions { + /// Create with recursive option + pub fn recursive() -> Self { + Self { + recursive: true, + ..Default::default() + } + } + + /// Create with ignore_existing option + pub fn ignore_existing() -> Self { + Self { + ignore_existing: true, + ..Default::default() + } + } +} + +/// Options for remove_node +#[derive(Debug, Clone, Default)] +pub struct RemoveNodeOptions { + /// Remove recursively + pub recursive: bool, + /// Don't fail if node doesn't exist + pub force: bool, +} + +impl RemoveNodeOptions { + /// Create with recursive option + pub fn recursive() -> Self { + Self { + recursive: true, + ..Default::default() + } + } +} + +/// Options for exists +#[derive(Debug, Clone, Default)] +pub struct ExistsOptions {} + +/// Options for get_node +#[derive(Debug, Clone, Default)] +pub struct GetNodeOptions { + /// Attributes to fetch + pub attributes: Option>, + /// Maximum size of the returned value + pub max_size: Option, +} + +/// Options for set_node +#[derive(Debug, Clone, Default)] +pub struct SetNodeOptions { + /// Create parent directories if needed + pub recursive: bool, + /// Overwrite existing node + pub force: bool, +} + +/// Options for list_node +#[derive(Debug, Clone, Default)] +pub struct ListNodeOptions { + /// Attributes to fetch for each child + pub attributes: Option>, + /// Maximum number of children to return + pub max_size: Option, +} + +/// Options for copy_node +#[derive(Debug, Clone, Default)] +pub struct CopyNodeOptions { + /// Copy recursively + pub recursive: bool, + /// Overwrite destination if exists + pub force: bool, + /// Preserve account ownership + pub preserve_account: bool, + /// Preserve expiration time + pub preserve_expiration_time: bool, +} + +/// Options for move_node +#[derive(Debug, Clone, Default)] +pub struct MoveNodeOptions { + /// Move recursively + pub recursive: bool, + /// Overwrite destination if exists + pub force: bool, + /// Preserve account ownership + pub preserve_account: bool, + /// Preserve expiration time + pub preserve_expiration_time: bool, +} + +/// Options for link_node +#[derive(Debug, Clone, Default)] +pub struct LinkNodeOptions { + /// Create parent directories if needed + pub recursive: bool, + /// Overwrite existing link + pub force: bool, + /// Don't fail if link already exists + pub ignore_existing: bool, +} + +// Internal parameter structs for API calls + +#[derive(Serialize)] +pub(crate) struct CreateNodeParams<'a> { + pub path: &'a str, + #[serde(rename = "type")] + pub r#type: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recursive: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub ignore_existing: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub force: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub attributes: Option>, +} + +#[derive(Serialize)] +pub(crate) struct RemoveNodeParams<'a> { + pub path: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recursive: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub force: bool, +} + +#[derive(Serialize)] +pub(crate) struct ExistsParams<'a> { + pub path: &'a str, +} + +#[derive(Serialize)] +pub(crate) struct GetNodeParams<'a> { + pub path: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub attributes: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_size: Option, +} + +#[derive(Serialize)] +pub(crate) struct SetNodeParams<'a> { + pub path: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recursive: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub force: bool, +} + +#[derive(Serialize)] +pub(crate) struct ListNodeParams<'a> { + pub path: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub attributes: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_size: Option, +} + +#[derive(Serialize)] +pub(crate) struct CopyNodeParams<'a> { + pub source_path: &'a str, + pub destination_path: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recursive: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub force: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub preserve_account: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub preserve_expiration_time: bool, +} + +#[derive(Serialize)] +pub(crate) struct MoveNodeParams<'a> { + pub source_path: &'a str, + pub destination_path: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recursive: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub force: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub preserve_account: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub preserve_expiration_time: bool, +} + +#[derive(Serialize)] +pub(crate) struct LinkNodeParams<'a> { + pub target_path: &'a str, + pub link_path: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recursive: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub force: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub ignore_existing: bool, +} diff --git a/yt/rust/ytsaurus-client/src/error.rs b/yt/rust/ytsaurus-client/src/error.rs new file mode 100644 index 000000000000..a3e91696beaf --- /dev/null +++ b/yt/rust/ytsaurus-client/src/error.rs @@ -0,0 +1,300 @@ +//! Error types for YTsaurus client. + +use std::collections::HashMap; +use std::fmt; +use ytsaurus_yson::YsonValue; + +/// YTsaurus error codes +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(i32)] +pub enum ErrorCode { + /// Generic error + Generic = 1, + /// Cancelled + Canceled = 2, + /// Timeout + Timeout = 3, + + // Cypress errors (500-599) + /// Node does not exist + ResolveError = 500, + /// Node already exists + AlreadyExists = 501, + /// Maximum child count exceeded + MaxChildCountViolation = 502, + /// Maximum string length exceeded + MaxStringLengthViolation = 503, + /// Maximum attribute size exceeded + MaxAttributeSizeViolation = 504, + /// Maximum key length exceeded + MaxKeyLengthViolation = 505, + + // Transaction errors (400-499) + /// No such transaction + NoSuchTransaction = 11000, + /// Nested transaction type mismatch + NestedTransactionTypeMismatch = 11001, + /// Transaction lock conflict + ConcurrentTransactionLockConflict = 402, + + // RPC errors (100-199) + /// Transport error + TransportError = 100, + /// Protocol error + ProtocolError = 101, + /// No such service + NoSuchService = 102, + /// No such method + NoSuchMethod = 103, + /// Unavailable + Unavailable = 105, + /// Request queue size limit exceeded + RequestQueueSizeLimitExceeded = 108, + + // Proxy errors (2100+) + /// Proxy banned + ProxyBanned = 2100, + + // Chunk errors (700-799) + /// All replicas are unavailable + ChunkUnavailable = 716, + + // Tablet errors (1700-1799) + /// Tablet not mounted + TabletNotMounted = 1702, + + /// Unknown error code + Unknown = -1, +} + +impl ErrorCode { + /// Create from integer code + pub fn from_code(code: i32) -> Self { + match code { + 1 => ErrorCode::Generic, + 2 => ErrorCode::Canceled, + 3 => ErrorCode::Timeout, + 500 => ErrorCode::ResolveError, + 501 => ErrorCode::AlreadyExists, + 502 => ErrorCode::MaxChildCountViolation, + 503 => ErrorCode::MaxStringLengthViolation, + 504 => ErrorCode::MaxAttributeSizeViolation, + 505 => ErrorCode::MaxKeyLengthViolation, + 11000 => ErrorCode::NoSuchTransaction, + 11001 => ErrorCode::NestedTransactionTypeMismatch, + 402 => ErrorCode::ConcurrentTransactionLockConflict, + 100 => ErrorCode::TransportError, + 101 => ErrorCode::ProtocolError, + 102 => ErrorCode::NoSuchService, + 103 => ErrorCode::NoSuchMethod, + 105 => ErrorCode::Unavailable, + 108 => ErrorCode::RequestQueueSizeLimitExceeded, + 2100 => ErrorCode::ProxyBanned, + 716 => ErrorCode::ChunkUnavailable, + 1702 => ErrorCode::TabletNotMounted, + _ => ErrorCode::Unknown, + } + } + + /// Get the integer code + pub fn as_code(&self) -> i32 { + *self as i32 + } +} + +impl fmt::Display for ErrorCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}({})", self, self.as_code()) + } +} + +/// YTsaurus error +#[derive(Debug, Clone)] +pub struct YtError { + /// Error code + pub code: ErrorCode, + /// Error message + pub message: String, + /// Additional attributes + pub attributes: HashMap, + /// Inner errors (error chain) + pub inner_errors: Vec, +} + +impl YtError { + /// Create a new error + pub fn new(code: ErrorCode, message: impl Into) -> Self { + Self { + code, + message: message.into(), + attributes: HashMap::new(), + inner_errors: Vec::new(), + } + } + + /// Create a generic error + pub fn generic(message: impl Into) -> Self { + Self::new(ErrorCode::Generic, message) + } + + /// Add an attribute + pub fn with_attribute(mut self, key: impl Into, value: YsonValue) -> Self { + self.attributes.insert(key.into(), value); + self + } + + /// Add an inner error + pub fn with_inner(mut self, inner: YtError) -> Self { + self.inner_errors.push(inner); + self + } + + /// Check if the error tree contains a specific error code + pub fn contains_code(&self, code: ErrorCode) -> bool { + if self.code == code { + return true; + } + self.inner_errors.iter().any(|e| e.contains_code(code)) + } + + /// Find the first error with a specific code + pub fn find_code(&self, code: ErrorCode) -> Option<&YtError> { + if self.code == code { + return Some(self); + } + for inner in &self.inner_errors { + if let Some(found) = inner.find_code(code) { + return Some(found); + } + } + None + } + + /// Check if this is a resolve error (path not found) + pub fn is_resolve_error(&self) -> bool { + self.contains_code(ErrorCode::ResolveError) + } + + /// Check if this is an "already exists" error + pub fn is_already_exists(&self) -> bool { + self.contains_code(ErrorCode::AlreadyExists) + } + + /// Check if this is a transaction lock conflict + pub fn is_lock_conflict(&self) -> bool { + self.contains_code(ErrorCode::ConcurrentTransactionLockConflict) + } + + /// Check if this error is retriable + pub fn is_retriable(&self) -> bool { + self.contains_code(ErrorCode::RequestQueueSizeLimitExceeded) + || self.contains_code(ErrorCode::Unavailable) + || self.contains_code(ErrorCode::ProxyBanned) + || self.contains_code(ErrorCode::TabletNotMounted) + || self.contains_code(ErrorCode::ChunkUnavailable) + || self.contains_code(ErrorCode::TransportError) + } + + /// Parse from YSON value (from X-YT-Error header or response) + pub fn from_yson(value: &YsonValue) -> Option { + let map = value.as_map()?; + + let code = map + .get(b"code".as_ref()) + .and_then(|v| v.as_i64()) + .map(|c| ErrorCode::from_code(c as i32)) + .unwrap_or(ErrorCode::Generic); + + let message = map + .get(b"message".as_ref()) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let mut error = YtError::new(code, message); + + // Parse attributes + if let Some(attrs) = map.get(b"attributes".as_ref()).and_then(|v| v.as_map()) { + for (key, value) in attrs { + if let Ok(key_str) = std::str::from_utf8(key) { + error.attributes.insert(key_str.to_string(), value.clone()); + } + } + } + + // Parse inner errors + if let Some(inner_list) = map.get(b"inner_errors".as_ref()).and_then(|v| v.as_list()) { + for inner_value in inner_list { + if let Some(inner_error) = YtError::from_yson(inner_value) { + error.inner_errors.push(inner_error); + } + } + } + + Some(error) + } +} + +impl std::error::Error for YtError {} + +impl fmt::Display for YtError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message)?; + if !self.inner_errors.is_empty() { + write!(f, ": {}", self.inner_errors[0])?; + } + Ok(()) + } +} + +impl From for YtError { + fn from(e: reqwest::Error) -> Self { + YtError::new(ErrorCode::TransportError, e.to_string()) + } +} + +impl From for YtError { + fn from(e: ytsaurus_yson::Error) -> Self { + YtError::new(ErrorCode::ProtocolError, e.to_string()) + } +} + +impl From for YtError { + fn from(e: std::io::Error) -> Self { + YtError::new(ErrorCode::TransportError, e.to_string()) + } +} + +/// Result type alias for YTsaurus operations +pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_code_conversion() { + assert_eq!(ErrorCode::from_code(500), ErrorCode::ResolveError); + assert_eq!(ErrorCode::from_code(501), ErrorCode::AlreadyExists); + assert_eq!(ErrorCode::from_code(9999), ErrorCode::Unknown); + } + + #[test] + fn test_contains_code() { + let inner = YtError::new(ErrorCode::ResolveError, "path not found"); + let outer = YtError::new(ErrorCode::Generic, "operation failed").with_inner(inner); + + assert!(outer.contains_code(ErrorCode::Generic)); + assert!(outer.contains_code(ErrorCode::ResolveError)); + assert!(!outer.contains_code(ErrorCode::AlreadyExists)); + } + + #[test] + fn test_is_retriable() { + let retriable = YtError::new(ErrorCode::Unavailable, "service unavailable"); + assert!(retriable.is_retriable()); + + let not_retriable = YtError::new(ErrorCode::ResolveError, "not found"); + assert!(!not_retriable.is_retriable()); + } +} diff --git a/yt/rust/ytsaurus-client/src/lib.rs b/yt/rust/ytsaurus-client/src/lib.rs new file mode 100644 index 000000000000..a5cccd6481ad --- /dev/null +++ b/yt/rust/ytsaurus-client/src/lib.rs @@ -0,0 +1,43 @@ +//! YTsaurus Rust client. +//! +//! This crate provides a Rust client for YTsaurus, a distributed storage and +//! processing platform for big data. +//! +//! # Example +//! +//! ```rust,no_run +//! use ytsaurus_client::{YtClient, YPath}; +//! +//! #[tokio::main] +//! async fn main() -> Result<(), Box> { +//! // Create client from environment (YT_PROXY, YT_TOKEN) +//! let client = YtClient::from_env()?; +//! +//! // Read a value from Cypress +//! let cluster_name: String = client +//! .get_node(&YPath::new("//sys/@cluster_name"), None) +//! .await?; +//! println!("Connected to: {}", cluster_name); +//! +//! Ok(()) +//! } +//! ``` + +mod client; +mod config; +mod auth; +mod error; +mod path; +mod cypress; +mod table; + +pub use client::YtClient; +pub use config::{ClientConfig, ClientBuilder}; +pub use auth::{Credentials, TokenCredentials, BearerCredentials}; +pub use error::{YtError, ErrorCode, Result}; +pub use path::YPath; +pub use cypress::{NodeType, NodeId, CreateNodeOptions, RemoveNodeOptions, GetNodeOptions, SetNodeOptions, ListNodeOptions, CopyNodeOptions, MoveNodeOptions, ExistsOptions}; +pub use table::{TableReader, TableWriter, ReadTableOptions, WriteTableOptions}; + +/// Re-export ytsaurus-yson for convenience +pub use ytsaurus_yson as yson; diff --git a/yt/rust/ytsaurus-client/src/path.rs b/yt/rust/ytsaurus-client/src/path.rs new file mode 100644 index 000000000000..1586934effab --- /dev/null +++ b/yt/rust/ytsaurus-client/src/path.rs @@ -0,0 +1,209 @@ +//! YPath - paths in YTsaurus Cypress. + +use std::fmt; + +/// A path in the YTsaurus Cypress tree. +/// +/// Paths start with `//` and use `/` as separator. +/// Attributes are accessed with `/@` suffix. +/// +/// # Examples +/// +/// ``` +/// use ytsaurus_client::YPath; +/// +/// let path = YPath::new("//home/user/table"); +/// let attr_path = path.attribute("row_count"); +/// assert_eq!(attr_path.as_str(), "//home/user/table/@row_count"); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct YPath(String); + +impl YPath { + /// Create a new path + pub fn new(path: impl Into) -> Self { + let path = path.into(); + // Ensure path starts with // + if path.starts_with("//") { + Self(path) + } else if path.starts_with('/') { + Self(format!("/{}", path)) + } else { + Self(format!("//{}", path)) + } + } + + /// Create a path for the root + pub fn root() -> Self { + Self("//".to_string()) + } + + /// Get the path as a string slice + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Join with a child component + pub fn join(&self, component: &str) -> Self { + if self.0.ends_with('/') { + Self(format!("{}{}", self.0, component)) + } else { + Self(format!("{}/{}", self.0, component)) + } + } + + /// Get an attribute path (//path/@attr) + pub fn attribute(&self, attr: &str) -> Self { + Self(format!("{}/@{}", self.0, attr)) + } + + /// Get the parent path + pub fn parent(&self) -> Option { + let path = self.0.trim_end_matches('/'); + if path == "/" || path == "//" { + return None; + } + path.rfind('/').map(|pos| { + let parent = &path[..pos]; + if parent.is_empty() || parent == "/" { + Self("//".to_string()) + } else { + Self(parent.to_string()) + } + }) + } + + /// Get the last component of the path + pub fn name(&self) -> Option<&str> { + let path = self.0.trim_end_matches('/'); + if path == "/" || path == "//" { + return None; + } + let name = path.rsplit('/').next()?; + if name.is_empty() { + None + } else { + Some(name) + } + } + + /// Check if this is a root path + pub fn is_root(&self) -> bool { + self.0 == "//" || self.0 == "/" + } + + /// Check if this is an attribute access path + pub fn is_attribute(&self) -> bool { + self.0.contains("/@") + } + + /// Add a range specification for table reading + pub fn with_range(self, start: Option, end: Option) -> YPathWithRange { + YPathWithRange { + path: self, + start, + end, + } + } +} + +impl fmt::Display for YPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From<&str> for YPath { + fn from(s: &str) -> Self { + YPath::new(s) + } +} + +impl From for YPath { + fn from(s: String) -> Self { + YPath::new(s) + } +} + +impl AsRef for YPath { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// A path with an optional row range +#[derive(Debug, Clone)] +pub struct YPathWithRange { + /// The base path + pub path: YPath, + /// Start row index (inclusive) + pub start: Option, + /// End row index (exclusive) + pub end: Option, +} + +impl YPathWithRange { + /// Get the path string with range suffix + pub fn to_string_with_range(&self) -> String { + match (self.start, self.end) { + (None, None) => self.path.0.clone(), + (Some(start), None) => format!("{}[#{}:]", self.path.0, start), + (None, Some(end)) => format!("{}[:#{}]", self.path.0, end), + (Some(start), Some(end)) => format!("{}[#{}:#{}]", self.path.0, start, end), + } + } +} + +impl fmt::Display for YPathWithRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_string_with_range()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new() { + assert_eq!(YPath::new("//home/user").as_str(), "//home/user"); + assert_eq!(YPath::new("/home/user").as_str(), "//home/user"); + assert_eq!(YPath::new("home/user").as_str(), "//home/user"); + } + + #[test] + fn test_join() { + let path = YPath::new("//home/user"); + assert_eq!(path.join("table").as_str(), "//home/user/table"); + } + + #[test] + fn test_attribute() { + let path = YPath::new("//home/user/table"); + assert_eq!(path.attribute("row_count").as_str(), "//home/user/table/@row_count"); + } + + #[test] + fn test_parent() { + let path = YPath::new("//home/user/table"); + assert_eq!(path.parent().unwrap().as_str(), "//home/user"); + assert_eq!(path.parent().unwrap().parent().unwrap().as_str(), "//home"); + assert_eq!(path.parent().unwrap().parent().unwrap().parent().unwrap().as_str(), "//"); + assert!(YPath::root().parent().is_none()); + } + + #[test] + fn test_name() { + assert_eq!(YPath::new("//home/user/table").name(), Some("table")); + assert_eq!(YPath::root().name(), None); + } + + #[test] + fn test_with_range() { + let path = YPath::new("//table"); + assert_eq!(path.clone().with_range(Some(0), Some(100)).to_string(), "//table[#0:#100]"); + assert_eq!(path.clone().with_range(Some(10), None).to_string(), "//table[#10:]"); + assert_eq!(path.clone().with_range(None, Some(50)).to_string(), "//table[:#50]"); + assert_eq!(path.with_range(None, None).to_string(), "//table"); + } +} diff --git a/yt/rust/ytsaurus-client/src/table.rs b/yt/rust/ytsaurus-client/src/table.rs new file mode 100644 index 000000000000..2621d7d9adc4 --- /dev/null +++ b/yt/rust/ytsaurus-client/src/table.rs @@ -0,0 +1,494 @@ +//! Table read/write operations. + +use crate::client::YtClient; +use crate::error::{ErrorCode, Result, YtError}; +use crate::path::YPath; +use futures::Stream; +use pin_project_lite::pin_project; +use reqwest::header::{HeaderMap, HeaderValue}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::marker::PhantomData; +use std::pin::Pin; +use std::task::{Context, Poll}; +use uuid::Uuid; +use ytsaurus_yson::{from_binary, to_binary}; + +/// Options for reading a table +#[derive(Debug, Clone, Default)] +pub struct ReadTableOptions { + /// Columns to read (None means all) + pub columns: Option>, + /// Row range start (inclusive) + pub start_row: Option, + /// Row range end (exclusive) + pub end_row: Option, + /// Unordered read (faster but no guaranteed order) + pub unordered: bool, +} + +/// Options for writing a table +#[derive(Debug, Clone, Default)] +pub struct WriteTableOptions { + /// Append to existing table + pub append: bool, + /// Table schema (for creating new table) + pub schema: Option, +} + +/// Table schema +#[derive(Debug, Clone, Serialize)] +pub struct TableSchema { + /// Columns + pub columns: Vec, + /// Strict schema (no extra columns allowed) + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub strict: bool, + /// Unique keys + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub unique_keys: bool, +} + +/// Column schema +#[derive(Debug, Clone, Serialize)] +pub struct ColumnSchema { + /// Column name + pub name: String, + /// Column type + #[serde(rename = "type")] + pub column_type: String, + /// Sort order (for key columns) + #[serde(skip_serializing_if = "Option::is_none")] + pub sort_order: Option, + /// Required (non-nullable) + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub required: bool, +} + +impl TableSchema { + /// Create a new schema with columns + pub fn new(columns: Vec) -> Self { + Self { + columns, + strict: true, + unique_keys: false, + } + } +} + +impl ColumnSchema { + /// Create a new column schema + pub fn new(name: impl Into, column_type: impl Into) -> Self { + Self { + name: name.into(), + column_type: column_type.into(), + sort_order: None, + required: false, + } + } + + /// Make this a key column + pub fn key(mut self) -> Self { + self.sort_order = Some("ascending".to_string()); + self + } + + /// Make this column required + pub fn required(mut self) -> Self { + self.required = true; + self + } +} + +pin_project! { + /// Stream of table rows + pub struct TableReader { + data: Vec, + pos: usize, + _marker: PhantomData, + } +} + +impl TableReader { + pub(crate) async fn new( + client: &YtClient, + path: &YPath, + options: Option, + ) -> Result { + let options = options.unwrap_or_default(); + + let path_with_range = if options.start_row.is_some() || options.end_row.is_some() { + path.clone() + .with_range(options.start_row, options.end_row) + .to_string_with_range() + } else { + path.as_str().to_string() + }; + + // Build parameters as JSON for header (X-YT-Parameters expects JSON) + let mut params_json = format!("{{\"path\":\"{}\"}}", path_with_range.replace('"', "\\\"")); + if options.unordered || options.columns.is_some() { + let columns_json = match &options.columns { + Some(cols) => { + let cols_str: Vec = cols.iter().map(|c| format!("\"{}\"", c)).collect(); + format!(",\"columns\":[{}]", cols_str.join(",")) + } + None => String::new(), + }; + params_json = format!( + "{{\"path\":\"{}\",\"unordered\":{}{}}}", + path_with_range.replace('"', "\\\""), + options.unordered, + columns_json + ); + } + + let url = format!("{}/api/v4/read_table", client.config().base_url()); + let correlation_id = Uuid::new_v4().to_string(); + + let mut headers = HeaderMap::new(); + headers.insert( + "X-YT-Correlation-Id", + HeaderValue::from_str(&correlation_id).unwrap(), + ); + headers.insert( + "X-YT-Output-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "X-YT-Parameters", + HeaderValue::from_str(¶ms_json).unwrap(), + ); + + client.credentials.apply_headers(&mut headers); + + let response = client + .http_client + .get(&url) + .headers(headers) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let response_bytes = response.bytes().await?; + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new( + ErrorCode::Generic, + format!("HTTP error: {}", status), + )); + } + + let data = response.bytes().await?.to_vec(); + + Ok(Self { + data, + pos: 0, + _marker: PhantomData, + }) + } + + /// Collect all rows into a vector + pub async fn collect(mut self) -> Result> { + let mut rows = Vec::new(); + while let Some(row) = self.next_row()? { + rows.push(row); + } + Ok(rows) + } + + /// Get the next row + fn next_row(&mut self) -> Result> { + if self.pos >= self.data.len() { + return Ok(None); + } + + // Skip semicolon separator + if self.data[self.pos] == b';' { + self.pos += 1; + } + + if self.pos >= self.data.len() { + return Ok(None); + } + + // Find the end of this row (next semicolon or end of data) + let start = self.pos; + let mut depth = 0; + while self.pos < self.data.len() { + match self.data[self.pos] { + b'{' | b'[' | b'<' => depth += 1, + b'}' | b']' | b'>' => depth -= 1, + b';' if depth == 0 => break, + 0x01 => { + // Binary string - skip length + data + self.pos += 1; + let (len, consumed) = read_varint(&self.data[self.pos..])?; + self.pos += consumed + len as usize; + continue; + } + _ => {} + } + self.pos += 1; + } + + let row_data = &self.data[start..self.pos]; + let row: T = from_binary(row_data)?; + Ok(Some(row)) + } +} + +impl Stream for TableReader { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + + if *this.pos >= this.data.len() { + return Poll::Ready(None); + } + + // Skip semicolon separator + if this.data[*this.pos] == b';' { + *this.pos += 1; + } + + if *this.pos >= this.data.len() { + return Poll::Ready(None); + } + + // Find the end of this row + let start = *this.pos; + let mut depth = 0; + while *this.pos < this.data.len() { + match this.data[*this.pos] { + b'{' | b'[' | b'<' => depth += 1, + b'}' | b']' | b'>' => depth -= 1, + b';' if depth == 0 => break, + 0x01 => { + *this.pos += 1; + if let Ok((len, consumed)) = read_varint(&this.data[*this.pos..]) { + *this.pos += consumed + len as usize; + continue; + } + } + _ => {} + } + *this.pos += 1; + } + + let row_data = &this.data[start..*this.pos]; + match from_binary(row_data) { + Ok(row) => Poll::Ready(Some(Ok(row))), + Err(e) => Poll::Ready(Some(Err(e.into()))), + } + } +} + +/// Table writer for writing rows +pub struct TableWriter { + client_config: crate::config::ClientConfig, + http_client: reqwest::Client, + credentials: std::sync::Arc, + path: String, + append: bool, + buffer: Vec, + row_count: usize, + committed: bool, + _marker: PhantomData, +} + +impl TableWriter { + pub(crate) async fn new( + client: &YtClient, + path: &YPath, + options: Option, + ) -> Result { + use crate::cypress::{CreateNodeOptions, NodeType}; + + let options = options.unwrap_or_default(); + + // If not appending, create the table if it doesn't exist + if !options.append { + let mut create_opts = CreateNodeOptions::default(); + create_opts.ignore_existing = true; + let _ = client.create_node(path, NodeType::Table, Some(create_opts)).await; + } + + // Schema support can be added later with proper YSON serialization + let _ = &options.schema; // Acknowledge but don't use for now + + Ok(Self { + client_config: client.config().clone(), + http_client: client.http_client.clone(), + credentials: client.credentials.clone(), + path: if options.append { + format!("{}", path.as_str()) + } else { + path.as_str().to_string() + }, + append: options.append, + buffer: Vec::new(), + row_count: 0, + committed: false, + _marker: PhantomData, + }) + } + + /// Write a single row + pub fn write(&mut self, row: &T) -> Result<()> { + if self.row_count > 0 { + self.buffer.push(b';'); + } + let row_bytes = to_binary(row)?; + self.buffer.extend_from_slice(&row_bytes); + self.row_count += 1; + Ok(()) + } + + /// Write multiple rows + pub fn write_batch(&mut self, rows: &[T]) -> Result<()> { + for row in rows { + self.write(row)?; + } + Ok(()) + } + + /// Commit the write operation + pub async fn commit(mut self) -> Result<()> { + if self.committed { + return Ok(()); + } + + // Build parameters as JSON for header (X-YT-Parameters expects JSON) + let params_json = format!("{{\"path\":\"{}\"}}", self.path.replace('"', "\\\"")); + + let url = format!("{}/api/v4/write_table", self.client_config.base_url()); + let correlation_id = Uuid::new_v4().to_string(); + + let mut headers = HeaderMap::new(); + headers.insert( + "X-YT-Correlation-Id", + HeaderValue::from_str(&correlation_id).unwrap(), + ); + headers.insert( + "X-YT-Input-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "X-YT-Output-Format", + HeaderValue::from_static("\"yson\""), + ); + headers.insert( + "X-YT-Parameters", + HeaderValue::from_str(¶ms_json).unwrap(), + ); + headers.insert( + "Content-Type", + HeaderValue::from_static("application/x-yson"), + ); + + self.credentials.apply_headers(&mut headers); + + let response = self + .http_client + .put(&url) + .headers(headers) + .body(std::mem::take(&mut self.buffer)) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let response_bytes = response.bytes().await?; + if let Ok(error_value) = ytsaurus_yson::binary::reader::parse(&response_bytes) { + if let Some(error) = YtError::from_yson(&error_value) { + return Err(error); + } + } + return Err(YtError::new( + ErrorCode::Generic, + format!("HTTP error: {}", status), + )); + } + + self.committed = true; + Ok(()) + } + + /// Get the number of rows written so far + pub fn row_count(&self) -> usize { + self.row_count + } + + /// Rollback (discard) the write without committing + pub fn rollback(mut self) { + self.committed = true; // Prevent commit on drop + self.buffer.clear(); + } +} + +impl Drop for TableWriter { + fn drop(&mut self) { + if !self.committed && !self.buffer.is_empty() { + tracing::warn!( + "TableWriter dropped without commit, {} rows discarded", + self.row_count + ); + } + } +} + +/// Read a varint from bytes +fn read_varint(bytes: &[u8]) -> Result<(u64, usize)> { + let mut result: u64 = 0; + let mut shift = 0; + let mut pos = 0; + + loop { + if pos >= bytes.len() { + return Err(YtError::new(ErrorCode::ProtocolError, "unexpected EOF in varint")); + } + let byte = bytes[pos]; + pos += 1; + + result |= ((byte & 0x7F) as u64) << shift; + + if (byte & 0x80) == 0 { + break; + } + shift += 7; + } + + // Decode zigzag for signed length + let signed = ((result >> 1) as i64) ^ (-((result & 1) as i64)); + Ok((signed as u64, pos)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_column_schema() { + let col = ColumnSchema::new("id", "int64").key().required(); + assert_eq!(col.name, "id"); + assert_eq!(col.column_type, "int64"); + assert_eq!(col.sort_order, Some("ascending".to_string())); + assert!(col.required); + } + + #[test] + fn test_table_schema() { + let schema = TableSchema::new(vec![ + ColumnSchema::new("id", "int64").key(), + ColumnSchema::new("name", "string"), + ]); + assert_eq!(schema.columns.len(), 2); + assert!(schema.strict); + } +} diff --git a/yt/rust/ytsaurus-client/tests/integration_test.rs b/yt/rust/ytsaurus-client/tests/integration_test.rs new file mode 100644 index 000000000000..7dfc7100df34 --- /dev/null +++ b/yt/rust/ytsaurus-client/tests/integration_test.rs @@ -0,0 +1,305 @@ +//! Integration tests for YTsaurus Rust client. +//! +//! These tests require a running YTsaurus cluster. +//! Set YT_PROXY environment variable before running. + +use serde::{Deserialize, Serialize}; +use ytsaurus_client::{ + CreateNodeOptions, NodeType, RemoveNodeOptions, YPath, YtClient, +}; + +fn get_client() -> YtClient { + YtClient::builder() + .proxy("localhost:8000") + .build() + .expect("Failed to create client") +} + +#[tokio::test] +async fn test_cluster_connection() { + let client = get_client(); + + // Get cluster name + let cluster_name: String = client + .get_node(&YPath::new("//sys/@cluster_name"), None) + .await + .expect("Failed to get cluster name"); + + println!("Connected to cluster: {}", cluster_name); + assert!(!cluster_name.is_empty()); +} + +#[tokio::test] +async fn test_cypress_create_and_remove() { + let client = get_client(); + let path = YPath::new("//tmp/rust_test_create_remove"); + + // Clean up if exists + let _ = client + .remove_node(&path, Some(RemoveNodeOptions::recursive())) + .await; + + // Create a map node + let node_id = client + .create_node(&path, NodeType::Map, Some(CreateNodeOptions::recursive())) + .await + .expect("Failed to create node"); + + println!("Created node: {}", node_id); + assert!(!node_id.is_empty()); + + // Verify it exists + let exists = client.exists(&path, None).await.expect("Failed to check exists"); + assert!(exists); + + // Remove it + client + .remove_node(&path, None) + .await + .expect("Failed to remove node"); + + // Verify it's gone + let exists = client.exists(&path, None).await.expect("Failed to check exists"); + assert!(!exists); +} + +#[tokio::test] +async fn test_cypress_get_set() { + let client = get_client(); + let base_path = YPath::new("//tmp/rust_test_get_set"); + + // Clean up + let _ = client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await; + + // Create directory + client + .create_node( + &base_path, + NodeType::Map, + Some(CreateNodeOptions::recursive()), + ) + .await + .expect("Failed to create directory"); + + // Set a string value + let value_path = base_path.join("string_value"); + client + .set_node(&value_path, &"hello world", None) + .await + .expect("Failed to set string value"); + + // Get it back + let value: String = client + .get_node(&value_path, None) + .await + .expect("Failed to get string value"); + assert_eq!(value, "hello world"); + + // Set an integer value + let int_path = base_path.join("int_value"); + client + .set_node(&int_path, &42i64, None) + .await + .expect("Failed to set int value"); + + let int_value: i64 = client + .get_node(&int_path, None) + .await + .expect("Failed to get int value"); + assert_eq!(int_value, 42); + + // Clean up + client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await + .expect("Failed to cleanup"); +} + +#[tokio::test] +async fn test_cypress_list() { + let client = get_client(); + let base_path = YPath::new("//tmp/rust_test_list"); + + // Clean up + let _ = client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await; + + // Create directory with children + client + .create_node( + &base_path, + NodeType::Map, + Some(CreateNodeOptions::recursive()), + ) + .await + .expect("Failed to create directory"); + + client + .create_node(&base_path.join("child1"), NodeType::Map, None) + .await + .expect("Failed to create child1"); + + client + .create_node(&base_path.join("child2"), NodeType::Map, None) + .await + .expect("Failed to create child2"); + + client + .create_node(&base_path.join("child3"), NodeType::Map, None) + .await + .expect("Failed to create child3"); + + // List children + let children = client + .list_node(&base_path, None) + .await + .expect("Failed to list children"); + + println!("Children: {:?}", children); + assert_eq!(children.len(), 3); + assert!(children.contains(&"child1".to_string())); + assert!(children.contains(&"child2".to_string())); + assert!(children.contains(&"child3".to_string())); + + // Clean up + client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await + .expect("Failed to cleanup"); +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +struct TestRow { + id: i64, + name: String, + value: f64, +} + +#[tokio::test] +async fn test_table_write_read() { + let client = get_client(); + let table_path = YPath::new("//tmp/rust_test_table"); + + // Clean up + let _ = client + .remove_node(&table_path, Some(RemoveNodeOptions::recursive())) + .await; + + // Create test data + let rows = vec![ + TestRow { + id: 1, + name: "Alice".to_string(), + value: 1.5, + }, + TestRow { + id: 2, + name: "Bob".to_string(), + value: 2.5, + }, + TestRow { + id: 3, + name: "Charlie".to_string(), + value: 3.5, + }, + ]; + + // Write table + let mut writer = client + .write_table(&table_path, None) + .await + .expect("Failed to create table writer"); + + for row in &rows { + writer.write(row).expect("Failed to write row"); + } + writer.commit().await.expect("Failed to commit write"); + + println!("Wrote {} rows to {}", rows.len(), table_path); + + // Read table back + let reader = client + .read_table::(&table_path, None) + .await + .expect("Failed to create table reader"); + + let read_rows = reader.collect().await.expect("Failed to read rows"); + + println!("Read {} rows", read_rows.len()); + assert_eq!(read_rows.len(), rows.len()); + + for (i, row) in read_rows.iter().enumerate() { + println!(" Row {}: {:?}", i, row); + assert_eq!(row, &rows[i]); + } + + // Clean up + client + .remove_node(&table_path, Some(RemoveNodeOptions::recursive())) + .await + .expect("Failed to cleanup"); +} + +#[tokio::test] +async fn test_copy_move() { + let client = get_client(); + let base_path = YPath::new("//tmp/rust_test_copy_move"); + let src_path = base_path.join("source"); + let copy_path = base_path.join("copy"); + let move_path = base_path.join("moved"); + + // Clean up + let _ = client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await; + + // Create source + client + .create_node( + &src_path, + NodeType::Map, + Some(CreateNodeOptions::recursive()), + ) + .await + .expect("Failed to create source"); + + // Set a value in source + client + .set_node(&src_path.join("data"), &"test data", None) + .await + .expect("Failed to set data"); + + // Copy + client + .copy_node(&src_path, ©_path, None) + .await + .expect("Failed to copy"); + + // Verify copy has the data + let copy_data: String = client + .get_node(©_path.join("data"), None) + .await + .expect("Failed to get copy data"); + assert_eq!(copy_data, "test data"); + + // Move copy to new location + client + .move_node(©_path, &move_path, None) + .await + .expect("Failed to move"); + + // Verify move succeeded + let exists_old = client.exists(©_path, None).await.unwrap(); + let exists_new = client.exists(&move_path, None).await.unwrap(); + assert!(!exists_old); + assert!(exists_new); + + // Clean up + client + .remove_node(&base_path, Some(RemoveNodeOptions::recursive())) + .await + .expect("Failed to cleanup"); +} diff --git a/yt/rust/ytsaurus-yson/Cargo.toml b/yt/rust/ytsaurus-yson/Cargo.toml new file mode 100644 index 000000000000..c3829f587b32 --- /dev/null +++ b/yt/rust/ytsaurus-yson/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ytsaurus-yson" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "YSON serialization format for YTsaurus" +keywords = ["ytsaurus", "yson", "serialization"] +categories = ["encoding", "parser-implementations"] + +[dependencies] +serde.workspace = true +thiserror.workspace = true +bytes.workspace = true +indexmap.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/yt/rust/ytsaurus-yson/src/binary/mod.rs b/yt/rust/ytsaurus-yson/src/binary/mod.rs new file mode 100644 index 000000000000..46915fc11c8b --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/binary/mod.rs @@ -0,0 +1,8 @@ +//! Binary YSON format support. + +pub mod varint; +pub mod reader; +pub mod writer; + +pub use reader::BinaryReader; +pub use writer::BinaryWriter; diff --git a/yt/rust/ytsaurus-yson/src/binary/reader.rs b/yt/rust/ytsaurus-yson/src/binary/reader.rs new file mode 100644 index 000000000000..c4b65a711421 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/binary/reader.rs @@ -0,0 +1,544 @@ +//! Binary YSON reader/parser. + +use crate::markers::*; +use crate::value::YsonValue; +use crate::{Error, Result}; +use indexmap::IndexMap; +use super::varint::{read_varint_u64, read_varint_i64}; + +/// Maximum nesting depth to prevent stack overflow +const MAX_DEPTH: usize = 256; + +/// Binary YSON reader +pub struct BinaryReader<'a> { + data: &'a [u8], + pos: usize, + depth: usize, +} + +impl<'a> BinaryReader<'a> { + /// Create a new binary reader from a byte slice + pub fn new(data: &'a [u8]) -> Self { + Self { + data, + pos: 0, + depth: 0, + } + } + + /// Parse a complete YSON node + pub fn parse_node(&mut self) -> Result { + self.parse_value() + } + + /// Parse a list fragment (semicolon-separated values) + pub fn parse_list_fragment(&mut self) -> Result> { + let mut items = Vec::new(); + while self.pos < self.data.len() { + items.push(self.parse_value()?); + // Skip optional semicolon + if self.pos < self.data.len() && self.data[self.pos] == ITEM_SEPARATOR { + self.pos += 1; + } + } + Ok(items) + } + + /// Check if there's more data to read + pub fn has_more(&self) -> bool { + self.pos < self.data.len() + } + + /// Get current position + pub fn position(&self) -> usize { + self.pos + } + + fn parse_value(&mut self) -> Result { + if self.pos >= self.data.len() { + return Err(Error::UnexpectedEof); + } + + if self.depth > MAX_DEPTH { + return Err(Error::NestingTooDeep); + } + + let marker = self.data[self.pos]; + + match marker { + // Entity + ENTITY => { + self.pos += 1; + Ok(YsonValue::Entity) + } + + // Binary string + STRING => { + self.pos += 1; + let len = read_varint_i64(self.data, &mut self.pos)? as usize; + if self.pos + len > self.data.len() { + return Err(Error::UnexpectedEof); + } + let bytes = self.data[self.pos..self.pos + len].to_vec(); + self.pos += len; + Ok(YsonValue::String(bytes)) + } + + // Binary int64 + INT64 => { + self.pos += 1; + let value = read_varint_i64(self.data, &mut self.pos)?; + Ok(YsonValue::Int64(value)) + } + + // Binary uint64 + UINT64 => { + self.pos += 1; + let value = read_varint_u64(self.data, &mut self.pos)?; + Ok(YsonValue::Uint64(value)) + } + + // Binary double + DOUBLE => { + self.pos += 1; + if self.pos + 8 > self.data.len() { + return Err(Error::UnexpectedEof); + } + let bytes: [u8; 8] = self.data[self.pos..self.pos + 8] + .try_into() + .map_err(|_| Error::UnexpectedEof)?; + self.pos += 8; + Ok(YsonValue::Double(f64::from_le_bytes(bytes))) + } + + // Binary false + FALSE => { + self.pos += 1; + Ok(YsonValue::Boolean(false)) + } + + // Binary true + TRUE => { + self.pos += 1; + Ok(YsonValue::Boolean(true)) + } + + // List + LIST_BEGIN => { + self.pos += 1; + self.depth += 1; + let mut items = Vec::new(); + loop { + if self.pos >= self.data.len() { + return Err(Error::UnexpectedEof); + } + if self.data[self.pos] == LIST_END { + self.pos += 1; + break; + } + items.push(self.parse_value()?); + // Skip optional semicolon + if self.pos < self.data.len() && self.data[self.pos] == ITEM_SEPARATOR { + self.pos += 1; + } + } + self.depth -= 1; + Ok(YsonValue::List(items)) + } + + // Map + MAP_BEGIN => { + self.pos += 1; + self.depth += 1; + let mut map = IndexMap::new(); + loop { + if self.pos >= self.data.len() { + return Err(Error::UnexpectedEof); + } + if self.data[self.pos] == MAP_END { + self.pos += 1; + break; + } + let key = self.parse_string_key()?; + if self.pos >= self.data.len() || self.data[self.pos] != KEY_VALUE_SEPARATOR { + return Err(Error::UnexpectedToken { + expected: "'='", + found: format!("byte 0x{:02x}", self.data.get(self.pos).unwrap_or(&0)), + }); + } + self.pos += 1; // skip '=' + let value = self.parse_value()?; + map.insert(key, value); + // Skip optional semicolon + if self.pos < self.data.len() && self.data[self.pos] == ITEM_SEPARATOR { + self.pos += 1; + } + } + self.depth -= 1; + Ok(YsonValue::Map(map)) + } + + // Attributes - skip them and parse the value + ATTRIBUTES_BEGIN => { + self.pos += 1; + self.depth += 1; + // Parse and discard attributes + loop { + if self.pos >= self.data.len() { + return Err(Error::UnexpectedEof); + } + if self.data[self.pos] == ATTRIBUTES_END { + self.pos += 1; + break; + } + let _key = self.parse_string_key()?; + if self.pos >= self.data.len() || self.data[self.pos] != KEY_VALUE_SEPARATOR { + return Err(Error::UnexpectedToken { + expected: "'='", + found: format!("byte 0x{:02x}", self.data.get(self.pos).unwrap_or(&0)), + }); + } + self.pos += 1; + let _value = self.parse_value()?; + if self.pos < self.data.len() && self.data[self.pos] == ITEM_SEPARATOR { + self.pos += 1; + } + } + self.depth -= 1; + // Now parse the actual value + self.parse_value() + } + + // Text format tokens - try to parse as text + b'"' => self.parse_text_string(), + b'%' => self.parse_text_special(), + b'0'..=b'9' | b'-' | b'+' => self.parse_text_number(), + c if c.is_ascii_alphabetic() || c == b'_' => self.parse_text_identifier(), + + _ => Err(Error::InvalidMarker(marker)), + } + } + + fn parse_string_key(&mut self) -> Result> { + if self.pos >= self.data.len() { + return Err(Error::UnexpectedEof); + } + + match self.data[self.pos] { + STRING => { + self.pos += 1; + let len = read_varint_i64(self.data, &mut self.pos)? as usize; + if self.pos + len > self.data.len() { + return Err(Error::UnexpectedEof); + } + let bytes = self.data[self.pos..self.pos + len].to_vec(); + self.pos += len; + Ok(bytes) + } + b'"' => { + self.pos += 1; + self.parse_quoted_string() + } + c if c.is_ascii_alphabetic() || c == b'_' => { + let start = self.pos; + while self.pos < self.data.len() { + let c = self.data[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'-' { + self.pos += 1; + } else { + break; + } + } + Ok(self.data[start..self.pos].to_vec()) + } + c => Err(Error::UnexpectedToken { + expected: "string key", + found: format!("byte 0x{:02x}", c), + }), + } + } + + fn parse_text_string(&mut self) -> Result { + self.pos += 1; // skip opening quote + let bytes = self.parse_quoted_string()?; + Ok(YsonValue::String(bytes)) + } + + fn parse_quoted_string(&mut self) -> Result> { + let mut result = Vec::new(); + while self.pos < self.data.len() { + let c = self.data[self.pos]; + self.pos += 1; + match c { + b'"' => return Ok(result), + b'\\' => { + if self.pos >= self.data.len() { + return Err(Error::UnexpectedEof); + } + let escaped = self.data[self.pos]; + self.pos += 1; + match escaped { + b'"' => result.push(b'"'), + b'\\' => result.push(b'\\'), + b'n' => result.push(b'\n'), + b'r' => result.push(b'\r'), + b't' => result.push(b'\t'), + b'x' => { + if self.pos + 2 > self.data.len() { + return Err(Error::UnexpectedEof); + } + let hex = std::str::from_utf8(&self.data[self.pos..self.pos + 2]) + .map_err(|_| Error::InvalidUtf8)?; + let byte = u8::from_str_radix(hex, 16) + .map_err(|_| Error::SyntaxError(format!("invalid hex escape: {}", hex)))?; + result.push(byte); + self.pos += 2; + } + b'0'..=b'7' => { + // Octal escape + let mut value = (escaped - b'0') as u32; + for _ in 0..2 { + if self.pos < self.data.len() { + let c = self.data[self.pos]; + if (b'0'..=b'7').contains(&c) { + value = value * 8 + (c - b'0') as u32; + self.pos += 1; + } else { + break; + } + } + } + result.push(value as u8); + } + _ => result.push(escaped), + } + } + _ => result.push(c), + } + } + Err(Error::UnexpectedEof) + } + + fn parse_text_special(&mut self) -> Result { + self.pos += 1; // skip '%' + let start = self.pos; + while self.pos < self.data.len() && self.data[self.pos].is_ascii_alphabetic() { + self.pos += 1; + } + let keyword = std::str::from_utf8(&self.data[start..self.pos]) + .map_err(|_| Error::InvalidUtf8)?; + match keyword { + "true" => Ok(YsonValue::Boolean(true)), + "false" => Ok(YsonValue::Boolean(false)), + "nan" => Ok(YsonValue::Double(f64::NAN)), + "inf" | "+inf" => Ok(YsonValue::Double(f64::INFINITY)), + "-inf" => Ok(YsonValue::Double(f64::NEG_INFINITY)), + _ => Err(Error::SyntaxError(format!("unknown keyword: %{}", keyword))), + } + } + + fn parse_text_number(&mut self) -> Result { + let start = self.pos; + let mut is_float = false; + let mut is_unsigned = false; + + // Handle sign + if self.pos < self.data.len() && (self.data[self.pos] == b'-' || self.data[self.pos] == b'+') { + self.pos += 1; + } + + // Parse digits + while self.pos < self.data.len() && self.data[self.pos].is_ascii_digit() { + self.pos += 1; + } + + // Check for decimal point or exponent + if self.pos < self.data.len() { + if self.data[self.pos] == b'.' { + is_float = true; + self.pos += 1; + while self.pos < self.data.len() && self.data[self.pos].is_ascii_digit() { + self.pos += 1; + } + } + if self.pos < self.data.len() && (self.data[self.pos] == b'e' || self.data[self.pos] == b'E') { + is_float = true; + self.pos += 1; + if self.pos < self.data.len() && (self.data[self.pos] == b'-' || self.data[self.pos] == b'+') { + self.pos += 1; + } + while self.pos < self.data.len() && self.data[self.pos].is_ascii_digit() { + self.pos += 1; + } + } + if self.pos < self.data.len() && self.data[self.pos] == b'u' { + is_unsigned = true; + self.pos += 1; + } + } + + let num_str = std::str::from_utf8(&self.data[start..self.pos - if is_unsigned { 1 } else { 0 }]) + .map_err(|_| Error::InvalidUtf8)?; + + if is_float { + let value: f64 = num_str.parse() + .map_err(|_| Error::SyntaxError(format!("invalid number: {}", num_str)))?; + Ok(YsonValue::Double(value)) + } else if is_unsigned { + let value: u64 = num_str.parse() + .map_err(|_| Error::SyntaxError(format!("invalid unsigned number: {}", num_str)))?; + Ok(YsonValue::Uint64(value)) + } else { + let value: i64 = num_str.parse() + .map_err(|_| Error::SyntaxError(format!("invalid number: {}", num_str)))?; + Ok(YsonValue::Int64(value)) + } + } + + fn parse_text_identifier(&mut self) -> Result { + let start = self.pos; + while self.pos < self.data.len() { + let c = self.data[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'-' { + self.pos += 1; + } else { + break; + } + } + Ok(YsonValue::String(self.data[start..self.pos].to_vec())) + } +} + +/// Parse binary YSON from bytes +pub fn parse(data: &[u8]) -> Result { + let mut reader = BinaryReader::new(data); + reader.parse_node() +} + +/// Parse a list fragment from bytes +pub fn parse_list_fragment(data: &[u8]) -> Result> { + let mut reader = BinaryReader::new(data); + reader.parse_list_fragment() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_entity() { + let data = [ENTITY]; + let value = parse(&data).unwrap(); + assert_eq!(value, YsonValue::Entity); + } + + #[test] + fn test_parse_boolean() { + assert_eq!(parse(&[FALSE]).unwrap(), YsonValue::Boolean(false)); + assert_eq!(parse(&[TRUE]).unwrap(), YsonValue::Boolean(true)); + } + + #[test] + fn test_parse_int64() { + // 0 + let data = [INT64, 0x00]; + assert_eq!(parse(&data).unwrap(), YsonValue::Int64(0)); + + // 1 + let data = [INT64, 0x02]; // zigzag(1) = 2 + assert_eq!(parse(&data).unwrap(), YsonValue::Int64(1)); + + // -1 + let data = [INT64, 0x01]; // zigzag(-1) = 1 + assert_eq!(parse(&data).unwrap(), YsonValue::Int64(-1)); + + // 42 + let data = [INT64, 0x54]; // zigzag(42) = 84 = 0x54 + assert_eq!(parse(&data).unwrap(), YsonValue::Int64(42)); + } + + #[test] + fn test_parse_uint64() { + let data = [UINT64, 0x2A]; // 42 + assert_eq!(parse(&data).unwrap(), YsonValue::Uint64(42)); + } + + #[test] + fn test_parse_double() { + let value = 3.14f64; + let mut data = vec![DOUBLE]; + data.extend_from_slice(&value.to_le_bytes()); + assert_eq!(parse(&data).unwrap(), YsonValue::Double(3.14)); + } + + #[test] + fn test_parse_string() { + // "hello" (5 bytes) + let data = [STRING, 0x0A, b'h', b'e', b'l', b'l', b'o']; // len = zigzag(5) = 10 = 0x0A + let value = parse(&data).unwrap(); + assert_eq!(value.as_str(), Some("hello")); + } + + #[test] + fn test_parse_list() { + // [1, 2, 3] + let data = [ + LIST_BEGIN, + INT64, 0x02, // 1 + ITEM_SEPARATOR, + INT64, 0x04, // 2 + ITEM_SEPARATOR, + INT64, 0x06, // 3 + LIST_END, + ]; + let value = parse(&data).unwrap(); + let list = value.as_list().unwrap(); + assert_eq!(list.len(), 3); + assert_eq!(list[0].as_i64(), Some(1)); + assert_eq!(list[1].as_i64(), Some(2)); + assert_eq!(list[2].as_i64(), Some(3)); + } + + #[test] + fn test_parse_map() { + // {a=1;b=2} + let data = [ + MAP_BEGIN, + STRING, 0x02, b'a', // key "a" (len=1, zigzag=2) + KEY_VALUE_SEPARATOR, + INT64, 0x02, // 1 + ITEM_SEPARATOR, + STRING, 0x02, b'b', // key "b" + KEY_VALUE_SEPARATOR, + INT64, 0x04, // 2 + MAP_END, + ]; + let value = parse(&data).unwrap(); + let map = value.as_map().unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get(b"a".as_ref()).unwrap().as_i64(), Some(1)); + assert_eq!(map.get(b"b".as_ref()).unwrap().as_i64(), Some(2)); + } + + #[test] + fn test_parse_text_format() { + // Text format parsing + assert_eq!(parse(b"42").unwrap(), YsonValue::Int64(42)); + assert_eq!(parse(b"-1").unwrap(), YsonValue::Int64(-1)); + assert_eq!(parse(b"100u").unwrap(), YsonValue::Uint64(100)); + assert_eq!(parse(b"3.14").unwrap(), YsonValue::Double(3.14)); + assert_eq!(parse(b"%true").unwrap(), YsonValue::Boolean(true)); + assert_eq!(parse(b"%false").unwrap(), YsonValue::Boolean(false)); + assert_eq!(parse(b"#").unwrap(), YsonValue::Entity); + } + + #[test] + fn test_parse_list_fragment() { + let data = b"\x02\x02;\x02\x04;\x02\x06"; // 1;2;3 in binary + let items = parse_list_fragment(data).unwrap(); + assert_eq!(items.len(), 3); + assert_eq!(items[0].as_i64(), Some(1)); + assert_eq!(items[1].as_i64(), Some(2)); + assert_eq!(items[2].as_i64(), Some(3)); + } +} diff --git a/yt/rust/ytsaurus-yson/src/binary/varint.rs b/yt/rust/ytsaurus-yson/src/binary/varint.rs new file mode 100644 index 000000000000..92e3fa4a53e7 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/binary/varint.rs @@ -0,0 +1,174 @@ +//! Varint and zigzag encoding for YSON binary format. +//! +//! YSON uses LEB128 (Little Endian Base 128) encoding for variable-length integers: +//! - Each byte has 7 data bits and 1 continuation bit (MSB) +//! - If MSB is set, more bytes follow +//! - For signed integers, zigzag encoding is used first + +use crate::{Error, Result}; + +/// Encode an unsigned 64-bit integer as varint +pub fn encode_varint_u64(value: u64) -> Vec { + let mut result = Vec::with_capacity(10); + let mut v = value; + while v >= 0x80 { + result.push(0x80 | (v & 0x7F) as u8); + v >>= 7; + } + result.push(v as u8); + result +} + +/// Write a varint to a buffer, returning bytes written +pub fn write_varint_u64(buf: &mut Vec, value: u64) { + let mut v = value; + while v >= 0x80 { + buf.push(0x80 | (v & 0x7F) as u8); + v >>= 7; + } + buf.push(v as u8); +} + +/// Decode a varint from bytes, returning (value, bytes_consumed) +pub fn decode_varint_u64(bytes: &[u8]) -> Result<(u64, usize)> { + let mut result: u64 = 0; + let mut shift = 0; + let mut pos = 0; + + loop { + if pos >= bytes.len() { + return Err(Error::UnexpectedEof); + } + let byte = bytes[pos]; + pos += 1; + + if shift >= 64 { + return Err(Error::IntegerOverflow); + } + + result |= ((byte & 0x7F) as u64) << shift; + + if (byte & 0x80) == 0 { + break; + } + shift += 7; + } + + Ok((result, pos)) +} + +/// Read a varint from a slice, advancing the position +pub fn read_varint_u64(bytes: &[u8], pos: &mut usize) -> Result { + let (value, consumed) = decode_varint_u64(&bytes[*pos..])?; + *pos += consumed; + Ok(value) +} + +/// Zigzag encode a signed 64-bit integer +/// Maps negative numbers to odd positive numbers: +/// 0 -> 0, -1 -> 1, 1 -> 2, -2 -> 3, 2 -> 4, ... +#[inline] +pub fn zigzag_encode(value: i64) -> u64 { + ((value << 1) ^ (value >> 63)) as u64 +} + +/// Zigzag decode an unsigned integer back to signed +#[inline] +pub fn zigzag_decode(value: u64) -> i64 { + ((value >> 1) as i64) ^ (-((value & 1) as i64)) +} + +/// Encode a signed 64-bit integer as varint with zigzag encoding +pub fn encode_varint_i64(value: i64) -> Vec { + encode_varint_u64(zigzag_encode(value)) +} + +/// Write a signed varint to a buffer +pub fn write_varint_i64(buf: &mut Vec, value: i64) { + write_varint_u64(buf, zigzag_encode(value)) +} + +/// Decode a signed varint from bytes +pub fn decode_varint_i64(bytes: &[u8]) -> Result<(i64, usize)> { + let (unsigned, consumed) = decode_varint_u64(bytes)?; + Ok((zigzag_decode(unsigned), consumed)) +} + +/// Read a signed varint from a slice +pub fn read_varint_i64(bytes: &[u8], pos: &mut usize) -> Result { + let unsigned = read_varint_u64(bytes, pos)?; + Ok(zigzag_decode(unsigned)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_varint_u64() { + // Single byte values + assert_eq!(encode_varint_u64(0), vec![0x00]); + assert_eq!(encode_varint_u64(1), vec![0x01]); + assert_eq!(encode_varint_u64(127), vec![0x7F]); + + // Two byte values + assert_eq!(encode_varint_u64(128), vec![0x80, 0x01]); + assert_eq!(encode_varint_u64(16383), vec![0xFF, 0x7F]); + + // Three byte values + assert_eq!(encode_varint_u64(16384), vec![0x80, 0x80, 0x01]); + + // Large value + let large = u64::MAX; + let encoded = encode_varint_u64(large); + let (decoded, _) = decode_varint_u64(&encoded).unwrap(); + assert_eq!(decoded, large); + } + + #[test] + fn test_varint_roundtrip() { + let test_values = [0u64, 1, 127, 128, 255, 256, 16383, 16384, u64::MAX]; + for &value in &test_values { + let encoded = encode_varint_u64(value); + let (decoded, consumed) = decode_varint_u64(&encoded).unwrap(); + assert_eq!(decoded, value); + assert_eq!(consumed, encoded.len()); + } + } + + #[test] + fn test_zigzag() { + assert_eq!(zigzag_encode(0), 0); + assert_eq!(zigzag_encode(-1), 1); + assert_eq!(zigzag_encode(1), 2); + assert_eq!(zigzag_encode(-2), 3); + assert_eq!(zigzag_encode(2), 4); + assert_eq!(zigzag_encode(i64::MAX), u64::MAX - 1); + assert_eq!(zigzag_encode(i64::MIN), u64::MAX); + + // Roundtrip + for value in [0i64, -1, 1, -2, 2, 100, -100, i64::MAX, i64::MIN] { + assert_eq!(zigzag_decode(zigzag_encode(value)), value); + } + } + + #[test] + fn test_varint_i64() { + let test_values = [0i64, -1, 1, -2, 2, 100, -100, i64::MAX, i64::MIN]; + for &value in &test_values { + let encoded = encode_varint_i64(value); + let (decoded, consumed) = decode_varint_i64(&encoded).unwrap(); + assert_eq!(decoded, value); + assert_eq!(consumed, encoded.len()); + } + } + + #[test] + fn test_decode_error() { + // Empty input + assert!(decode_varint_u64(&[]).is_err()); + + // Truncated input (continuation bit set but no more bytes) + assert!(decode_varint_u64(&[0x80]).is_err()); + } +} diff --git a/yt/rust/ytsaurus-yson/src/binary/writer.rs b/yt/rust/ytsaurus-yson/src/binary/writer.rs new file mode 100644 index 000000000000..3a4c81049f5e --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/binary/writer.rs @@ -0,0 +1,308 @@ +//! Binary YSON writer. + +use crate::markers::*; +use crate::value::YsonValue; +use super::varint::{write_varint_u64, write_varint_i64}; + +/// Binary YSON writer +pub struct BinaryWriter { + buffer: Vec, +} + +impl BinaryWriter { + /// Create a new binary writer + pub fn new() -> Self { + Self { + buffer: Vec::new(), + } + } + + /// Create a new binary writer with pre-allocated capacity + pub fn with_capacity(capacity: usize) -> Self { + Self { + buffer: Vec::with_capacity(capacity), + } + } + + /// Get the written bytes + pub fn into_bytes(self) -> Vec { + self.buffer + } + + /// Get a reference to the written bytes + pub fn as_bytes(&self) -> &[u8] { + &self.buffer + } + + /// Clear the buffer for reuse + pub fn clear(&mut self) { + self.buffer.clear(); + } + + /// Write a YSON value + pub fn write_value(&mut self, value: &YsonValue) { + match value { + YsonValue::Entity => self.write_entity(), + YsonValue::Boolean(b) => self.write_bool(*b), + YsonValue::Int64(n) => self.write_i64(*n), + YsonValue::Uint64(n) => self.write_u64(*n), + YsonValue::Double(n) => self.write_f64(*n), + YsonValue::String(s) => self.write_string(s), + YsonValue::List(items) => self.write_list(items), + YsonValue::Map(map) => self.write_map(map), + } + } + + /// Write entity (null) + pub fn write_entity(&mut self) { + self.buffer.push(ENTITY); + } + + /// Write a boolean + pub fn write_bool(&mut self, value: bool) { + self.buffer.push(if value { TRUE } else { FALSE }); + } + + /// Write a signed 64-bit integer + pub fn write_i64(&mut self, value: i64) { + self.buffer.push(INT64); + write_varint_i64(&mut self.buffer, value); + } + + /// Write an unsigned 64-bit integer + pub fn write_u64(&mut self, value: u64) { + self.buffer.push(UINT64); + write_varint_u64(&mut self.buffer, value); + } + + /// Write a double + pub fn write_f64(&mut self, value: f64) { + self.buffer.push(DOUBLE); + self.buffer.extend_from_slice(&value.to_le_bytes()); + } + + /// Write a string/bytes + pub fn write_string(&mut self, value: &[u8]) { + self.buffer.push(STRING); + write_varint_i64(&mut self.buffer, value.len() as i64); + self.buffer.extend_from_slice(value); + } + + /// Write a string from &str + pub fn write_str(&mut self, value: &str) { + self.write_string(value.as_bytes()); + } + + /// Write a list + pub fn write_list(&mut self, items: &[YsonValue]) { + self.buffer.push(LIST_BEGIN); + for (i, item) in items.iter().enumerate() { + if i > 0 { + self.buffer.push(ITEM_SEPARATOR); + } + self.write_value(item); + } + self.buffer.push(LIST_END); + } + + /// Write a map + pub fn write_map(&mut self, map: &indexmap::IndexMap, YsonValue>) { + self.buffer.push(MAP_BEGIN); + for (i, (key, value)) in map.iter().enumerate() { + if i > 0 { + self.buffer.push(ITEM_SEPARATOR); + } + self.write_string(key); + self.buffer.push(KEY_VALUE_SEPARATOR); + self.write_value(value); + } + self.buffer.push(MAP_END); + } + + /// Begin a list (for streaming) + pub fn begin_list(&mut self) { + self.buffer.push(LIST_BEGIN); + } + + /// End a list (for streaming) + pub fn end_list(&mut self) { + self.buffer.push(LIST_END); + } + + /// Begin a map (for streaming) + pub fn begin_map(&mut self) { + self.buffer.push(MAP_BEGIN); + } + + /// End a map (for streaming) + pub fn end_map(&mut self) { + self.buffer.push(MAP_END); + } + + /// Write an item separator + pub fn write_separator(&mut self) { + self.buffer.push(ITEM_SEPARATOR); + } + + /// Write a key-value separator + pub fn write_key_value_separator(&mut self) { + self.buffer.push(KEY_VALUE_SEPARATOR); + } + + /// Begin attributes + pub fn begin_attributes(&mut self) { + self.buffer.push(ATTRIBUTES_BEGIN); + } + + /// End attributes + pub fn end_attributes(&mut self) { + self.buffer.push(ATTRIBUTES_END); + } +} + +impl Default for BinaryWriter { + fn default() -> Self { + Self::new() + } +} + +/// Write a YSON value to bytes +pub fn write(value: &YsonValue) -> Vec { + let mut writer = BinaryWriter::new(); + writer.write_value(value); + writer.into_bytes() +} + +/// Write a list fragment (for table rows) +pub fn write_list_fragment(items: &[YsonValue]) -> Vec { + let mut writer = BinaryWriter::new(); + for (i, item) in items.iter().enumerate() { + if i > 0 { + writer.write_separator(); + } + writer.write_value(item); + } + writer.into_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary::reader::parse; + + #[test] + fn test_write_entity() { + let mut writer = BinaryWriter::new(); + writer.write_entity(); + assert_eq!(writer.as_bytes(), &[ENTITY]); + } + + #[test] + fn test_write_bool() { + let mut writer = BinaryWriter::new(); + writer.write_bool(true); + assert_eq!(writer.as_bytes(), &[TRUE]); + + writer.clear(); + writer.write_bool(false); + assert_eq!(writer.as_bytes(), &[FALSE]); + } + + #[test] + fn test_write_i64() { + let mut writer = BinaryWriter::new(); + writer.write_i64(0); + assert_eq!(writer.as_bytes(), &[INT64, 0x00]); + + writer.clear(); + writer.write_i64(1); + assert_eq!(writer.as_bytes(), &[INT64, 0x02]); + + writer.clear(); + writer.write_i64(-1); + assert_eq!(writer.as_bytes(), &[INT64, 0x01]); + } + + #[test] + fn test_write_u64() { + let mut writer = BinaryWriter::new(); + writer.write_u64(42); + assert_eq!(writer.as_bytes(), &[UINT64, 0x2A]); + } + + #[test] + fn test_write_f64() { + let mut writer = BinaryWriter::new(); + writer.write_f64(3.14); + let expected: Vec = std::iter::once(DOUBLE) + .chain(3.14f64.to_le_bytes()) + .collect(); + assert_eq!(writer.as_bytes(), &expected); + } + + #[test] + fn test_write_string() { + let mut writer = BinaryWriter::new(); + writer.write_str("hello"); + // STRING marker + length (5, zigzag = 10 = 0x0A) + "hello" + assert_eq!(writer.as_bytes(), &[STRING, 0x0A, b'h', b'e', b'l', b'l', b'o']); + } + + #[test] + fn test_roundtrip() { + let test_values = vec![ + YsonValue::Entity, + YsonValue::Boolean(true), + YsonValue::Boolean(false), + YsonValue::Int64(0), + YsonValue::Int64(42), + YsonValue::Int64(-100), + YsonValue::Int64(i64::MAX), + YsonValue::Int64(i64::MIN), + YsonValue::Uint64(0), + YsonValue::Uint64(42), + YsonValue::Uint64(u64::MAX), + YsonValue::Double(3.14), + YsonValue::Double(0.0), + YsonValue::Double(-1.5), + YsonValue::string("hello"), + YsonValue::string(""), + YsonValue::List(vec![ + YsonValue::Int64(1), + YsonValue::Int64(2), + YsonValue::Int64(3), + ]), + ]; + + for value in test_values { + let bytes = write(&value); + let parsed = parse(&bytes).unwrap(); + assert_eq!(parsed, value, "roundtrip failed for {:?}", value); + } + } + + #[test] + fn test_roundtrip_map() { + let mut map = indexmap::IndexMap::new(); + map.insert(b"name".to_vec(), YsonValue::string("Alice")); + map.insert(b"age".to_vec(), YsonValue::Int64(30)); + let value = YsonValue::Map(map); + + let bytes = write(&value); + let parsed = parse(&bytes).unwrap(); + assert_eq!(parsed, value); + } + + #[test] + fn test_write_list_fragment() { + let items = vec![ + YsonValue::Int64(1), + YsonValue::Int64(2), + YsonValue::Int64(3), + ]; + let bytes = write_list_fragment(&items); + // Should be: 1;2;3 without list brackets + assert!(!bytes.contains(&LIST_BEGIN)); + assert!(!bytes.contains(&LIST_END)); + } +} diff --git a/yt/rust/ytsaurus-yson/src/de.rs b/yt/rust/ytsaurus-yson/src/de.rs new file mode 100644 index 000000000000..bf8fe02d78bd --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/de.rs @@ -0,0 +1,741 @@ +//! Serde deserializer for YSON format. + +use crate::binary::varint::{read_varint_i64, read_varint_u64}; +use crate::markers::*; +use crate::{Error, Result}; +use serde::de::{self, DeserializeSeed, IntoDeserializer, MapAccess, SeqAccess, Visitor}; +use serde::Deserialize; + +/// Deserialize from binary YSON +pub fn from_binary<'de, T: Deserialize<'de>>(bytes: &'de [u8]) -> Result { + let mut deserializer = Deserializer::new(bytes); + T::deserialize(&mut deserializer) +} + +/// YSON deserializer +pub struct Deserializer<'de> { + input: &'de [u8], + pos: usize, +} + +impl<'de> Deserializer<'de> { + /// Create a new deserializer + pub fn new(input: &'de [u8]) -> Self { + Self { input, pos: 0 } + } + + fn peek(&self) -> Result { + self.input.get(self.pos).copied().ok_or(Error::UnexpectedEof) + } + + fn advance(&mut self) { + self.pos += 1; + } + + fn skip_attributes(&mut self) -> Result<()> { + if self.pos < self.input.len() && self.input[self.pos] == ATTRIBUTES_BEGIN { + self.advance(); + let mut depth = 1; + while depth > 0 && self.pos < self.input.len() { + match self.input[self.pos] { + ATTRIBUTES_BEGIN | MAP_BEGIN | LIST_BEGIN => depth += 1, + ATTRIBUTES_END | MAP_END | LIST_END => depth -= 1, + STRING => { + self.advance(); + let len = read_varint_i64(self.input, &mut self.pos)? as usize; + self.pos += len; + continue; + } + _ => {} + } + self.advance(); + } + } + Ok(()) + } + + fn read_string(&mut self) -> Result<&'de [u8]> { + let len = read_varint_i64(self.input, &mut self.pos)? as usize; + if self.pos + len > self.input.len() { + return Err(Error::UnexpectedEof); + } + let bytes = &self.input[self.pos..self.pos + len]; + self.pos += len; + Ok(bytes) + } +} + +impl<'de, 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { + type Error = Error; + + fn deserialize_any>(self, visitor: V) -> Result { + self.skip_attributes()?; + let marker = self.peek()?; + + match marker { + ENTITY => { + self.advance(); + visitor.visit_unit() + } + TRUE => { + self.advance(); + visitor.visit_bool(true) + } + FALSE => { + self.advance(); + visitor.visit_bool(false) + } + INT64 => { + self.advance(); + let value = read_varint_i64(self.input, &mut self.pos)?; + visitor.visit_i64(value) + } + UINT64 => { + self.advance(); + let value = read_varint_u64(self.input, &mut self.pos)?; + visitor.visit_u64(value) + } + DOUBLE => { + self.advance(); + if self.pos + 8 > self.input.len() { + return Err(Error::UnexpectedEof); + } + let bytes: [u8; 8] = self.input[self.pos..self.pos + 8] + .try_into() + .map_err(|_| Error::UnexpectedEof)?; + self.pos += 8; + visitor.visit_f64(f64::from_le_bytes(bytes)) + } + STRING => { + self.advance(); + let bytes = self.read_string()?; + if let Ok(s) = std::str::from_utf8(bytes) { + visitor.visit_borrowed_str(s) + } else { + visitor.visit_borrowed_bytes(bytes) + } + } + LIST_BEGIN => { + self.advance(); + let value = visitor.visit_seq(SeqDeserializer { de: self })?; + Ok(value) + } + MAP_BEGIN => { + self.advance(); + let value = visitor.visit_map(MapDeserializer { de: self })?; + Ok(value) + } + // Text format support + b'"' => { + self.advance(); + let start = self.pos; + while self.pos < self.input.len() && self.input[self.pos] != b'"' { + if self.input[self.pos] == b'\\' { + self.pos += 2; + } else { + self.pos += 1; + } + } + let bytes = &self.input[start..self.pos]; + self.advance(); // skip closing quote + if let Ok(s) = std::str::from_utf8(bytes) { + visitor.visit_borrowed_str(s) + } else { + visitor.visit_borrowed_bytes(bytes) + } + } + b'%' => { + self.advance(); + let start = self.pos; + while self.pos < self.input.len() && self.input[self.pos].is_ascii_alphabetic() { + self.pos += 1; + } + let keyword = std::str::from_utf8(&self.input[start..self.pos]) + .map_err(|_| Error::InvalidUtf8)?; + match keyword { + "true" => visitor.visit_bool(true), + "false" => visitor.visit_bool(false), + _ => Err(Error::SyntaxError(format!("unknown keyword: %{}", keyword))), + } + } + b'0'..=b'9' | b'-' | b'+' => { + let start = self.pos; + let mut is_float = false; + if self.input[self.pos] == b'-' || self.input[self.pos] == b'+' { + self.pos += 1; + } + while self.pos < self.input.len() && self.input[self.pos].is_ascii_digit() { + self.pos += 1; + } + if self.pos < self.input.len() && self.input[self.pos] == b'.' { + is_float = true; + self.pos += 1; + while self.pos < self.input.len() && self.input[self.pos].is_ascii_digit() { + self.pos += 1; + } + } + let num_str = std::str::from_utf8(&self.input[start..self.pos]) + .map_err(|_| Error::InvalidUtf8)?; + if is_float { + let value: f64 = num_str.parse().map_err(|_| { + Error::SyntaxError(format!("invalid number: {}", num_str)) + })?; + visitor.visit_f64(value) + } else { + let value: i64 = num_str.parse().map_err(|_| { + Error::SyntaxError(format!("invalid number: {}", num_str)) + })?; + visitor.visit_i64(value) + } + } + c if c.is_ascii_alphabetic() || c == b'_' => { + let start = self.pos; + while self.pos < self.input.len() { + let c = self.input[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'-' { + self.pos += 1; + } else { + break; + } + } + let s = std::str::from_utf8(&self.input[start..self.pos]) + .map_err(|_| Error::InvalidUtf8)?; + visitor.visit_borrowed_str(s) + } + _ => Err(Error::InvalidMarker(marker)), + } + } + + fn deserialize_bool>(self, visitor: V) -> Result { + self.skip_attributes()?; + match self.peek()? { + TRUE => { + self.advance(); + visitor.visit_bool(true) + } + FALSE => { + self.advance(); + visitor.visit_bool(false) + } + m => Err(Error::UnexpectedToken { + expected: "boolean", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_i8>(self, visitor: V) -> Result { + self.deserialize_i64(visitor) + } + + fn deserialize_i16>(self, visitor: V) -> Result { + self.deserialize_i64(visitor) + } + + fn deserialize_i32>(self, visitor: V) -> Result { + self.deserialize_i64(visitor) + } + + fn deserialize_i64>(self, visitor: V) -> Result { + self.skip_attributes()?; + match self.peek()? { + INT64 => { + self.advance(); + let value = read_varint_i64(self.input, &mut self.pos)?; + visitor.visit_i64(value) + } + UINT64 => { + self.advance(); + let value = read_varint_u64(self.input, &mut self.pos)?; + visitor.visit_i64(value as i64) + } + m => Err(Error::UnexpectedToken { + expected: "integer", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_u8>(self, visitor: V) -> Result { + self.deserialize_u64(visitor) + } + + fn deserialize_u16>(self, visitor: V) -> Result { + self.deserialize_u64(visitor) + } + + fn deserialize_u32>(self, visitor: V) -> Result { + self.deserialize_u64(visitor) + } + + fn deserialize_u64>(self, visitor: V) -> Result { + self.skip_attributes()?; + match self.peek()? { + UINT64 => { + self.advance(); + let value = read_varint_u64(self.input, &mut self.pos)?; + visitor.visit_u64(value) + } + INT64 => { + self.advance(); + let value = read_varint_i64(self.input, &mut self.pos)?; + visitor.visit_u64(value as u64) + } + m => Err(Error::UnexpectedToken { + expected: "unsigned integer", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_f32>(self, visitor: V) -> Result { + self.deserialize_f64(visitor) + } + + fn deserialize_f64>(self, visitor: V) -> Result { + self.skip_attributes()?; + match self.peek()? { + DOUBLE => { + self.advance(); + if self.pos + 8 > self.input.len() { + return Err(Error::UnexpectedEof); + } + let bytes: [u8; 8] = self.input[self.pos..self.pos + 8] + .try_into() + .map_err(|_| Error::UnexpectedEof)?; + self.pos += 8; + visitor.visit_f64(f64::from_le_bytes(bytes)) + } + INT64 => { + self.advance(); + let value = read_varint_i64(self.input, &mut self.pos)?; + visitor.visit_f64(value as f64) + } + m => Err(Error::UnexpectedToken { + expected: "double", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_char>(self, visitor: V) -> Result { + self.deserialize_str(visitor) + } + + fn deserialize_str>(self, visitor: V) -> Result { + self.skip_attributes()?; + match self.peek()? { + STRING => { + self.advance(); + let bytes = self.read_string()?; + let s = std::str::from_utf8(bytes).map_err(|_| Error::InvalidUtf8)?; + visitor.visit_borrowed_str(s) + } + // Text format: quoted string + b'"' => { + self.advance(); + let start = self.pos; + while self.pos < self.input.len() && self.input[self.pos] != b'"' { + if self.input[self.pos] == b'\\' { + self.pos += 2; + } else { + self.pos += 1; + } + } + let bytes = &self.input[start..self.pos]; + self.advance(); // skip closing quote + let s = std::str::from_utf8(bytes).map_err(|_| Error::InvalidUtf8)?; + visitor.visit_borrowed_str(s) + } + // Text format: unquoted identifier + c if c.is_ascii_alphabetic() || c == b'_' => { + let start = self.pos; + while self.pos < self.input.len() { + let c = self.input[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'-' { + self.pos += 1; + } else { + break; + } + } + let s = std::str::from_utf8(&self.input[start..self.pos]) + .map_err(|_| Error::InvalidUtf8)?; + visitor.visit_borrowed_str(s) + } + m => Err(Error::UnexpectedToken { + expected: "string", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_string>(self, visitor: V) -> Result { + self.deserialize_str(visitor) + } + + fn deserialize_bytes>(self, visitor: V) -> Result { + self.skip_attributes()?; + match self.peek()? { + STRING => { + self.advance(); + let bytes = self.read_string()?; + visitor.visit_borrowed_bytes(bytes) + } + m => Err(Error::UnexpectedToken { + expected: "bytes", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_byte_buf>(self, visitor: V) -> Result { + self.deserialize_bytes(visitor) + } + + fn deserialize_option>(self, visitor: V) -> Result { + self.skip_attributes()?; + if self.peek()? == ENTITY { + self.advance(); + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_unit>(self, visitor: V) -> Result { + self.skip_attributes()?; + if self.peek()? == ENTITY { + self.advance(); + visitor.visit_unit() + } else { + Err(Error::UnexpectedToken { + expected: "entity", + found: format!("marker 0x{:02x}", self.peek()?), + }) + } + } + + fn deserialize_unit_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + self.deserialize_unit(visitor) + } + + fn deserialize_newtype_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + visitor.visit_newtype_struct(self) + } + + fn deserialize_seq>(self, visitor: V) -> Result { + self.skip_attributes()?; + if self.peek()? != LIST_BEGIN { + return Err(Error::UnexpectedToken { + expected: "list", + found: format!("marker 0x{:02x}", self.peek()?), + }); + } + self.advance(); + visitor.visit_seq(SeqDeserializer { de: self }) + } + + fn deserialize_tuple>(self, _len: usize, visitor: V) -> Result { + self.deserialize_seq(visitor) + } + + fn deserialize_tuple_struct>( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result { + self.deserialize_seq(visitor) + } + + fn deserialize_map>(self, visitor: V) -> Result { + self.skip_attributes()?; + if self.peek()? != MAP_BEGIN { + return Err(Error::UnexpectedToken { + expected: "map", + found: format!("marker 0x{:02x}", self.peek()?), + }); + } + self.advance(); + visitor.visit_map(MapDeserializer { de: self }) + } + + fn deserialize_struct>( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result { + self.deserialize_map(visitor) + } + + fn deserialize_enum>( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result { + self.skip_attributes()?; + match self.peek()? { + STRING => { + // Unit variant + self.advance(); + let bytes = self.read_string()?; + let s = std::str::from_utf8(bytes).map_err(|_| Error::InvalidUtf8)?; + visitor.visit_enum(s.into_deserializer()) + } + MAP_BEGIN => { + // Newtype, tuple, or struct variant + self.advance(); + visitor.visit_enum(EnumDeserializer { de: self }) + } + m => Err(Error::UnexpectedToken { + expected: "enum", + found: format!("marker 0x{:02x}", m), + }), + } + } + + fn deserialize_identifier>(self, visitor: V) -> Result { + self.deserialize_str(visitor) + } + + fn deserialize_ignored_any>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } +} + +struct SeqDeserializer<'a, 'de: 'a> { + de: &'a mut Deserializer<'de>, +} + +impl<'de, 'a> SeqAccess<'de> for SeqDeserializer<'a, 'de> { + type Error = Error; + + fn next_element_seed>( + &mut self, + seed: T, + ) -> Result> { + if self.de.pos >= self.de.input.len() { + return Err(Error::UnexpectedEof); + } + if self.de.input[self.de.pos] == LIST_END { + self.de.advance(); + return Ok(None); + } + if self.de.input[self.de.pos] == ITEM_SEPARATOR { + self.de.advance(); + } + // Check for LIST_END again after skipping separator (for text YSON like `[1;2;]`) + if self.de.pos < self.de.input.len() && self.de.input[self.de.pos] == LIST_END { + self.de.advance(); + return Ok(None); + } + seed.deserialize(&mut *self.de).map(Some) + } +} + +struct MapDeserializer<'a, 'de: 'a> { + de: &'a mut Deserializer<'de>, +} + +impl<'de, 'a> MapAccess<'de> for MapDeserializer<'a, 'de> { + type Error = Error; + + fn next_key_seed>(&mut self, seed: K) -> Result> { + if self.de.pos >= self.de.input.len() { + return Err(Error::UnexpectedEof); + } + if self.de.input[self.de.pos] == MAP_END { + self.de.advance(); + return Ok(None); + } + if self.de.input[self.de.pos] == ITEM_SEPARATOR { + self.de.advance(); + } + // Check for MAP_END again after skipping separator (for text YSON like `{a=1;}`) + if self.de.pos < self.de.input.len() && self.de.input[self.de.pos] == MAP_END { + self.de.advance(); + return Ok(None); + } + seed.deserialize(&mut *self.de).map(Some) + } + + fn next_value_seed>(&mut self, seed: V) -> Result { + if self.de.pos >= self.de.input.len() || self.de.input[self.de.pos] != KEY_VALUE_SEPARATOR { + return Err(Error::UnexpectedToken { + expected: "'='", + found: format!("byte at pos {}", self.de.pos), + }); + } + self.de.advance(); + seed.deserialize(&mut *self.de) + } +} + +struct EnumDeserializer<'a, 'de: 'a> { + de: &'a mut Deserializer<'de>, +} + +impl<'de, 'a> de::EnumAccess<'de> for EnumDeserializer<'a, 'de> { + type Error = Error; + type Variant = Self; + + fn variant_seed>(self, seed: V) -> Result<(V::Value, Self::Variant)> { + let val = seed.deserialize(&mut *self.de)?; + if self.de.pos >= self.de.input.len() || self.de.input[self.de.pos] != KEY_VALUE_SEPARATOR { + return Err(Error::UnexpectedToken { + expected: "'='", + found: format!("byte at pos {}", self.de.pos), + }); + } + self.de.advance(); + Ok((val, self)) + } +} + +impl<'de, 'a> de::VariantAccess<'de> for EnumDeserializer<'a, 'de> { + type Error = Error; + + fn unit_variant(self) -> Result<()> { + Err(Error::Custom("expected unit variant".into())) + } + + fn newtype_variant_seed>(self, seed: T) -> Result { + let value = seed.deserialize(&mut *self.de)?; + if self.de.input[self.de.pos] == MAP_END { + self.de.advance(); + } + Ok(value) + } + + fn tuple_variant>(self, _len: usize, visitor: V) -> Result { + de::Deserializer::deserialize_seq(&mut *self.de, visitor) + } + + fn struct_variant>( + self, + _fields: &'static [&'static str], + visitor: V, + ) -> Result { + de::Deserializer::deserialize_map(&mut *self.de, visitor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ser::to_binary; + use serde::{Deserialize, Serialize}; + + #[test] + fn test_deserialize_primitives() { + assert_eq!(from_binary::(&[TRUE]).unwrap(), true); + assert_eq!(from_binary::(&[FALSE]).unwrap(), false); + assert_eq!(from_binary::(&[INT64, 0x00]).unwrap(), 0); + assert_eq!(from_binary::(&[INT64, 0x02]).unwrap(), 1); + assert_eq!(from_binary::(&[INT64, 0x01]).unwrap(), -1); + assert_eq!(from_binary::(&[UINT64, 0x2A]).unwrap(), 42); + } + + #[test] + fn test_deserialize_string() { + let bytes = [STRING, 0x0A, b'h', b'e', b'l', b'l', b'o']; // len=5, zigzag=10 + assert_eq!(from_binary::(&bytes).unwrap(), "hello"); + } + + #[test] + fn test_deserialize_vec() { + let bytes = [ + LIST_BEGIN, INT64, 0x02, ITEM_SEPARATOR, INT64, 0x04, ITEM_SEPARATOR, INT64, 0x06, + LIST_END, + ]; + let vec: Vec = from_binary(&bytes).unwrap(); + assert_eq!(vec, vec![1, 2, 3]); + } + + #[test] + fn test_deserialize_option() { + assert_eq!(from_binary::>(&[ENTITY]).unwrap(), None); + assert_eq!( + from_binary::>(&[INT64, 0x02]).unwrap(), + Some(1) + ); + } + + #[test] + fn test_roundtrip_struct() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Person { + name: String, + age: i64, + } + + let person = Person { + name: "Alice".into(), + age: 30, + }; + let bytes = to_binary(&person).unwrap(); + let decoded: Person = from_binary(&bytes).unwrap(); + assert_eq!(person, decoded); + } + + #[test] + fn test_roundtrip_nested() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Inner { + x: i64, + y: i64, + } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Outer { + name: String, + inner: Inner, + values: Vec, + } + + let value = Outer { + name: "test".into(), + inner: Inner { x: 1, y: 2 }, + values: vec![10, 20, 30], + }; + let bytes = to_binary(&value).unwrap(); + let decoded: Outer = from_binary(&bytes).unwrap(); + assert_eq!(value, decoded); + } + + #[test] + fn test_deserialize_text_struct() { + #[derive(Debug, PartialEq, Deserialize)] + struct ApiResponse { + value: String, + } + + // Text YSON from YTsaurus server + let input = b"{value=locasaurus;}"; + let result: ApiResponse = from_binary(input).unwrap(); + assert_eq!(result.value, "locasaurus"); + } + + #[test] + fn test_deserialize_text_string() { + // Unquoted identifier + let input = b"locasaurus"; + let result: String = from_binary(input).unwrap(); + assert_eq!(result, "locasaurus"); + + // Quoted string + let input2 = br#""hello world""#; + let result2: String = from_binary(input2).unwrap(); + assert_eq!(result2, "hello world"); + } +} diff --git a/yt/rust/ytsaurus-yson/src/error.rs b/yt/rust/ytsaurus-yson/src/error.rs new file mode 100644 index 000000000000..477f0c052d19 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/error.rs @@ -0,0 +1,77 @@ +//! Error types for YSON parsing and serialization. + +use std::fmt; + +/// YSON error type +#[derive(Debug)] +pub enum Error { + /// Unexpected end of input + UnexpectedEof, + /// Invalid marker byte + InvalidMarker(u8), + /// Invalid UTF-8 in string + InvalidUtf8, + /// Syntax error in text format + SyntaxError(String), + /// Unexpected token + UnexpectedToken { + expected: &'static str, + found: String, + }, + /// Nesting depth exceeded + NestingTooDeep, + /// Integer overflow + IntegerOverflow, + /// IO error + Io(std::io::Error), + /// Serde error + Custom(String), +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Io(e) => Some(e), + _ => None, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::UnexpectedEof => write!(f, "unexpected end of input"), + Error::InvalidMarker(m) => write!(f, "invalid marker byte: 0x{:02x}", m), + Error::InvalidUtf8 => write!(f, "invalid UTF-8 in string"), + Error::SyntaxError(msg) => write!(f, "syntax error: {}", msg), + Error::UnexpectedToken { expected, found } => { + write!(f, "expected {}, found {}", expected, found) + } + Error::NestingTooDeep => write!(f, "nesting depth exceeded maximum"), + Error::IntegerOverflow => write!(f, "integer overflow"), + Error::Io(e) => write!(f, "IO error: {}", e), + Error::Custom(msg) => write!(f, "{}", msg), + } + } +} + +impl From for Error { + fn from(e: std::io::Error) -> Self { + Error::Io(e) + } +} + +impl serde::de::Error for Error { + fn custom(msg: T) -> Self { + Error::Custom(msg.to_string()) + } +} + +impl serde::ser::Error for Error { + fn custom(msg: T) -> Self { + Error::Custom(msg.to_string()) + } +} + +/// Result type alias for YSON operations +pub type Result = std::result::Result; diff --git a/yt/rust/ytsaurus-yson/src/lib.rs b/yt/rust/ytsaurus-yson/src/lib.rs new file mode 100644 index 000000000000..e2d9193058a0 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/lib.rs @@ -0,0 +1,72 @@ +//! YSON serialization format for YTsaurus. +//! +//! YSON (YT-SON) is YTsaurus's primary data serialization format, similar to JSON +//! but with several enhancements: +//! - Binary representation for efficiency +//! - Attributes system (arbitrary metadata on any value) +//! - Three data types: Node, ListFragment, MapFragment +//! +//! # Example +//! +//! ```rust +//! use ytsaurus_yson::{YsonValue, to_binary, from_binary}; +//! use serde::{Serialize, Deserialize}; +//! +//! #[derive(Serialize, Deserialize, Debug, PartialEq)] +//! struct Row { +//! id: i64, +//! name: String, +//! } +//! +//! let row = Row { id: 42, name: "test".into() }; +//! let bytes = to_binary(&row).unwrap(); +//! let decoded: Row = from_binary(&bytes).unwrap(); +//! assert_eq!(row, decoded); +//! ``` + +pub mod binary; +pub mod text; +pub mod value; +mod de; +mod ser; +mod error; + +pub use error::{Error, Result}; +pub use value::{YsonValue, YsonType}; +pub use de::from_binary; +pub use ser::to_binary; + +/// Binary format markers +pub mod markers { + /// Binary string marker + pub const STRING: u8 = 0x01; + /// Binary int64 marker (zigzag encoded) + pub const INT64: u8 = 0x02; + /// Binary double marker (IEEE 754 little-endian) + pub const DOUBLE: u8 = 0x03; + /// Binary false marker + pub const FALSE: u8 = 0x04; + /// Binary true marker + pub const TRUE: u8 = 0x05; + /// Binary uint64 marker + pub const UINT64: u8 = 0x06; + + /// Entity (null) marker + pub const ENTITY: u8 = b'#'; + /// List begin marker + pub const LIST_BEGIN: u8 = b'['; + /// List end marker + pub const LIST_END: u8 = b']'; + /// Map begin marker + pub const MAP_BEGIN: u8 = b'{'; + /// Map end marker + pub const MAP_END: u8 = b'}'; + /// Attributes begin marker + pub const ATTRIBUTES_BEGIN: u8 = b'<'; + /// Attributes end marker + pub const ATTRIBUTES_END: u8 = b'>'; + /// Key-value separator + pub const KEY_VALUE_SEPARATOR: u8 = b'='; + /// Item separator + pub const ITEM_SEPARATOR: u8 = b';'; +} diff --git a/yt/rust/ytsaurus-yson/src/ser.rs b/yt/rust/ytsaurus-yson/src/ser.rs new file mode 100644 index 000000000000..2c8f077f10aa --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/ser.rs @@ -0,0 +1,422 @@ +//! Serde serializer for YSON format. + +use crate::binary::varint::{write_varint_i64, write_varint_u64}; +use crate::markers::*; +use crate::{Error, Result}; +use serde::ser::{self, Serialize}; + +/// Serialize a value to binary YSON +pub fn to_binary(value: &T) -> Result> { + let mut serializer = Serializer::new(); + value.serialize(&mut serializer)?; + Ok(serializer.into_bytes()) +} + +/// Serialize a value to binary YSON with pre-allocated capacity +pub fn to_binary_with_capacity(value: &T, capacity: usize) -> Result> { + let mut serializer = Serializer::with_capacity(capacity); + value.serialize(&mut serializer)?; + Ok(serializer.into_bytes()) +} + +/// YSON serializer +pub struct Serializer { + output: Vec, +} + +impl Serializer { + /// Create a new serializer + pub fn new() -> Self { + Self { output: Vec::new() } + } + + /// Create a new serializer with pre-allocated capacity + pub fn with_capacity(capacity: usize) -> Self { + Self { + output: Vec::with_capacity(capacity), + } + } + + /// Get the serialized bytes + pub fn into_bytes(self) -> Vec { + self.output + } +} + +impl Default for Serializer { + fn default() -> Self { + Self::new() + } +} + +impl<'a> ser::Serializer for &'a mut Serializer { + type Ok = (); + type Error = Error; + + type SerializeSeq = Self; + type SerializeTuple = Self; + type SerializeTupleStruct = Self; + type SerializeTupleVariant = Self; + type SerializeMap = Self; + type SerializeStruct = Self; + type SerializeStructVariant = Self; + + fn serialize_bool(self, v: bool) -> Result<()> { + self.output.push(if v { TRUE } else { FALSE }); + Ok(()) + } + + fn serialize_i8(self, v: i8) -> Result<()> { + self.serialize_i64(v as i64) + } + + fn serialize_i16(self, v: i16) -> Result<()> { + self.serialize_i64(v as i64) + } + + fn serialize_i32(self, v: i32) -> Result<()> { + self.serialize_i64(v as i64) + } + + fn serialize_i64(self, v: i64) -> Result<()> { + self.output.push(INT64); + write_varint_i64(&mut self.output, v); + Ok(()) + } + + fn serialize_u8(self, v: u8) -> Result<()> { + self.serialize_u64(v as u64) + } + + fn serialize_u16(self, v: u16) -> Result<()> { + self.serialize_u64(v as u64) + } + + fn serialize_u32(self, v: u32) -> Result<()> { + self.serialize_u64(v as u64) + } + + fn serialize_u64(self, v: u64) -> Result<()> { + self.output.push(UINT64); + write_varint_u64(&mut self.output, v); + Ok(()) + } + + fn serialize_f32(self, v: f32) -> Result<()> { + self.serialize_f64(v as f64) + } + + fn serialize_f64(self, v: f64) -> Result<()> { + self.output.push(DOUBLE); + self.output.extend_from_slice(&v.to_le_bytes()); + Ok(()) + } + + fn serialize_char(self, v: char) -> Result<()> { + let mut buf = [0u8; 4]; + let s = v.encode_utf8(&mut buf); + self.serialize_str(s) + } + + fn serialize_str(self, v: &str) -> Result<()> { + self.serialize_bytes(v.as_bytes()) + } + + fn serialize_bytes(self, v: &[u8]) -> Result<()> { + self.output.push(STRING); + write_varint_i64(&mut self.output, v.len() as i64); + self.output.extend_from_slice(v); + Ok(()) + } + + fn serialize_none(self) -> Result<()> { + self.serialize_unit() + } + + fn serialize_some(self, value: &T) -> Result<()> { + value.serialize(self) + } + + fn serialize_unit(self) -> Result<()> { + self.output.push(ENTITY); + Ok(()) + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + self.serialize_unit() + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<()> { + self.serialize_str(variant) + } + + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result<()> { + value.serialize(self) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result<()> { + self.output.push(MAP_BEGIN); + self.serialize_str(variant)?; + self.output.push(KEY_VALUE_SEPARATOR); + value.serialize(&mut *self)?; + self.output.push(MAP_END); + Ok(()) + } + + fn serialize_seq(self, _len: Option) -> Result { + self.output.push(LIST_BEGIN); + Ok(self) + } + + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_seq(Some(len)) + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + len: usize, + ) -> Result { + self.serialize_seq(Some(len)) + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + _len: usize, + ) -> Result { + self.output.push(MAP_BEGIN); + self.serialize_str(variant)?; + self.output.push(KEY_VALUE_SEPARATOR); + self.output.push(LIST_BEGIN); + Ok(self) + } + + fn serialize_map(self, _len: Option) -> Result { + self.output.push(MAP_BEGIN); + Ok(self) + } + + fn serialize_struct(self, _name: &'static str, len: usize) -> Result { + self.serialize_map(Some(len)) + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + _len: usize, + ) -> Result { + self.output.push(MAP_BEGIN); + self.serialize_str(variant)?; + self.output.push(KEY_VALUE_SEPARATOR); + self.output.push(MAP_BEGIN); + Ok(self) + } +} + +impl<'a> ser::SerializeSeq for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> { + if self.output.last() != Some(&LIST_BEGIN) { + self.output.push(ITEM_SEPARATOR); + } + value.serialize(&mut **self) + } + + fn end(self) -> Result<()> { + self.output.push(LIST_END); + Ok(()) + } +} + +impl<'a> ser::SerializeTuple for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result<()> { + ser::SerializeSeq::end(self) + } +} + +impl<'a> ser::SerializeTupleStruct for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> { + ser::SerializeSeq::serialize_element(self, value) + } + + fn end(self) -> Result<()> { + ser::SerializeSeq::end(self) + } +} + +impl<'a> ser::SerializeTupleVariant for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> { + if self.output.last() != Some(&LIST_BEGIN) { + self.output.push(ITEM_SEPARATOR); + } + value.serialize(&mut **self) + } + + fn end(self) -> Result<()> { + self.output.push(LIST_END); + self.output.push(MAP_END); + Ok(()) + } +} + +impl<'a> ser::SerializeMap for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<()> { + if self.output.last() != Some(&MAP_BEGIN) { + self.output.push(ITEM_SEPARATOR); + } + key.serialize(&mut **self) + } + + fn serialize_value(&mut self, value: &T) -> Result<()> { + self.output.push(KEY_VALUE_SEPARATOR); + value.serialize(&mut **self) + } + + fn end(self) -> Result<()> { + self.output.push(MAP_END); + Ok(()) + } +} + +impl<'a> ser::SerializeStruct for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<()> { + if self.output.last() != Some(&MAP_BEGIN) { + self.output.push(ITEM_SEPARATOR); + } + key.serialize(&mut **self)?; + self.output.push(KEY_VALUE_SEPARATOR); + value.serialize(&mut **self) + } + + fn end(self) -> Result<()> { + self.output.push(MAP_END); + Ok(()) + } +} + +impl<'a> ser::SerializeStructVariant for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<()> { + if self.output.last() != Some(&MAP_BEGIN) { + self.output.push(ITEM_SEPARATOR); + } + key.serialize(&mut **self)?; + self.output.push(KEY_VALUE_SEPARATOR); + value.serialize(&mut **self) + } + + fn end(self) -> Result<()> { + self.output.push(MAP_END); + self.output.push(MAP_END); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Serialize; + + #[test] + fn test_serialize_primitives() { + assert_eq!(to_binary(&true).unwrap(), vec![TRUE]); + assert_eq!(to_binary(&false).unwrap(), vec![FALSE]); + assert_eq!(to_binary(&42i64).unwrap(), vec![INT64, 0x54]); // zigzag(42) = 84 + assert_eq!(to_binary(&42u64).unwrap(), vec![UINT64, 0x2A]); + } + + #[test] + fn test_serialize_string() { + let bytes = to_binary(&"hello").unwrap(); + assert_eq!(bytes[0], STRING); + assert_eq!(bytes[1], 0x0A); // zigzag(5) = 10 + assert_eq!(&bytes[2..], b"hello"); + } + + #[test] + fn test_serialize_vec() { + let vec = vec![1i64, 2i64, 3i64]; + let bytes = to_binary(&vec).unwrap(); + assert_eq!(bytes[0], LIST_BEGIN); + assert_eq!(*bytes.last().unwrap(), LIST_END); + } + + #[test] + fn test_serialize_struct() { + #[derive(Serialize)] + struct Person { + name: String, + age: i64, + } + + let person = Person { + name: "Alice".into(), + age: 30, + }; + let bytes = to_binary(&person).unwrap(); + assert_eq!(bytes[0], MAP_BEGIN); + assert_eq!(*bytes.last().unwrap(), MAP_END); + } + + #[test] + fn test_serialize_option() { + assert_eq!(to_binary(&None::).unwrap(), vec![ENTITY]); + assert_eq!(to_binary(&Some(42i64)).unwrap(), vec![INT64, 0x54]); + } + + #[test] + fn test_serialize_unit() { + assert_eq!(to_binary(&()).unwrap(), vec![ENTITY]); + } +} diff --git a/yt/rust/ytsaurus-yson/src/text/mod.rs b/yt/rust/ytsaurus-yson/src/text/mod.rs new file mode 100644 index 000000000000..d6d0f55ab5c4 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/text/mod.rs @@ -0,0 +1,7 @@ +//! Text YSON format support. + +pub mod reader; +pub mod writer; + +pub use reader::TextReader; +pub use writer::TextWriter; diff --git a/yt/rust/ytsaurus-yson/src/text/reader.rs b/yt/rust/ytsaurus-yson/src/text/reader.rs new file mode 100644 index 000000000000..6c42c7e7d6bd --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/text/reader.rs @@ -0,0 +1,443 @@ +//! Text YSON reader/parser. + +use crate::value::YsonValue; +use crate::{Error, Result}; +use indexmap::IndexMap; + +/// Maximum nesting depth +const MAX_DEPTH: usize = 256; + +/// Text YSON reader +pub struct TextReader<'a> { + input: &'a str, + pos: usize, + depth: usize, +} + +impl<'a> TextReader<'a> { + /// Create a new text reader + pub fn new(input: &'a str) -> Self { + Self { + input, + pos: 0, + depth: 0, + } + } + + /// Parse a complete YSON node + pub fn parse_node(&mut self) -> Result { + self.skip_whitespace(); + self.parse_value() + } + + /// Check if there's more data + pub fn has_more(&self) -> bool { + self.pos < self.input.len() + } + + fn skip_whitespace(&mut self) { + while self.pos < self.input.len() { + let c = self.input.as_bytes()[self.pos]; + if c == b' ' || c == b'\t' || c == b'\n' || c == b'\r' { + self.pos += 1; + } else { + break; + } + } + } + + fn peek(&self) -> Option { + self.input[self.pos..].chars().next() + } + + fn advance(&mut self) { + if let Some(c) = self.peek() { + self.pos += c.len_utf8(); + } + } + + fn parse_value(&mut self) -> Result { + self.skip_whitespace(); + + if self.pos >= self.input.len() { + return Err(Error::UnexpectedEof); + } + + if self.depth > MAX_DEPTH { + return Err(Error::NestingTooDeep); + } + + // Handle attributes + if self.peek() == Some('<') { + self.advance(); + self.depth += 1; + // Parse and discard attributes + self.parse_map_content('>')?; + self.depth -= 1; + self.skip_whitespace(); + } + + let c = self.peek().ok_or(Error::UnexpectedEof)?; + + match c { + '#' => { + self.advance(); + Ok(YsonValue::Entity) + } + '%' => self.parse_special(), + '"' => self.parse_quoted_string(), + '[' => self.parse_list(), + '{' => self.parse_map(), + '-' | '+' | '0'..='9' => self.parse_number(), + c if c.is_ascii_alphabetic() || c == '_' => self.parse_identifier(), + c => Err(Error::SyntaxError(format!("unexpected character: {}", c))), + } + } + + fn parse_special(&mut self) -> Result { + self.advance(); // skip '%' + let start = self.pos; + while self.pos < self.input.len() { + let c = self.input.as_bytes()[self.pos]; + if c.is_ascii_alphabetic() || c == b'-' || c == b'+' { + self.pos += 1; + } else { + break; + } + } + let keyword = &self.input[start..self.pos]; + match keyword { + "true" => Ok(YsonValue::Boolean(true)), + "false" => Ok(YsonValue::Boolean(false)), + "nan" => Ok(YsonValue::Double(f64::NAN)), + "inf" | "+inf" => Ok(YsonValue::Double(f64::INFINITY)), + "-inf" => Ok(YsonValue::Double(f64::NEG_INFINITY)), + _ => Err(Error::SyntaxError(format!("unknown keyword: %{}", keyword))), + } + } + + fn parse_quoted_string(&mut self) -> Result { + self.advance(); // skip opening quote + let bytes = self.parse_string_content('"')?; + Ok(YsonValue::String(bytes)) + } + + fn parse_string_content(&mut self, terminator: char) -> Result> { + let mut result = Vec::new(); + while self.pos < self.input.len() { + let c = self.peek().ok_or(Error::UnexpectedEof)?; + if c == terminator { + self.advance(); + return Ok(result); + } + if c == '\\' { + self.advance(); + let escaped = self.peek().ok_or(Error::UnexpectedEof)?; + self.advance(); + match escaped { + '"' => result.push(b'"'), + '\\' => result.push(b'\\'), + 'n' => result.push(b'\n'), + 'r' => result.push(b'\r'), + 't' => result.push(b'\t'), + 'x' => { + let hex: String = self.input[self.pos..].chars().take(2).collect(); + if hex.len() != 2 { + return Err(Error::UnexpectedEof); + } + let byte = u8::from_str_radix(&hex, 16) + .map_err(|_| Error::SyntaxError(format!("invalid hex escape: {}", hex)))?; + result.push(byte); + self.pos += 2; + } + '0'..='7' => { + let mut value = (escaped as u8 - b'0') as u32; + for _ in 0..2 { + if let Some(c) = self.peek() { + if ('0'..='7').contains(&c) { + value = value * 8 + (c as u8 - b'0') as u32; + self.advance(); + } else { + break; + } + } + } + result.push(value as u8); + } + c => result.extend_from_slice(c.to_string().as_bytes()), + } + } else { + result.extend_from_slice(c.to_string().as_bytes()); + self.advance(); + } + } + Err(Error::UnexpectedEof) + } + + fn parse_number(&mut self) -> Result { + let start = self.pos; + let mut is_float = false; + let mut is_unsigned = false; + + // Handle sign + if matches!(self.peek(), Some('-') | Some('+')) { + self.advance(); + } + + // Parse integer part + while matches!(self.peek(), Some('0'..='9')) { + self.advance(); + } + + // Check for decimal point + if self.peek() == Some('.') { + is_float = true; + self.advance(); + while matches!(self.peek(), Some('0'..='9')) { + self.advance(); + } + } + + // Check for exponent + if matches!(self.peek(), Some('e') | Some('E')) { + is_float = true; + self.advance(); + if matches!(self.peek(), Some('-') | Some('+')) { + self.advance(); + } + while matches!(self.peek(), Some('0'..='9')) { + self.advance(); + } + } + + // Check for unsigned suffix + if self.peek() == Some('u') { + is_unsigned = true; + self.advance(); + } + + let num_str = if is_unsigned { + &self.input[start..self.pos - 1] + } else { + &self.input[start..self.pos] + }; + + if is_float { + let value: f64 = num_str.parse() + .map_err(|_| Error::SyntaxError(format!("invalid number: {}", num_str)))?; + Ok(YsonValue::Double(value)) + } else if is_unsigned { + let value: u64 = num_str.parse() + .map_err(|_| Error::SyntaxError(format!("invalid unsigned number: {}", num_str)))?; + Ok(YsonValue::Uint64(value)) + } else { + let value: i64 = num_str.parse() + .map_err(|_| Error::SyntaxError(format!("invalid number: {}", num_str)))?; + Ok(YsonValue::Int64(value)) + } + } + + fn parse_identifier(&mut self) -> Result { + let start = self.pos; + while self.pos < self.input.len() { + let c = self.input.as_bytes()[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'-' { + self.pos += 1; + } else { + break; + } + } + Ok(YsonValue::String(self.input[start..self.pos].as_bytes().to_vec())) + } + + fn parse_list(&mut self) -> Result { + self.advance(); // skip '[' + self.depth += 1; + let mut items = Vec::new(); + + loop { + self.skip_whitespace(); + if self.peek() == Some(']') { + self.advance(); + break; + } + items.push(self.parse_value()?); + self.skip_whitespace(); + if self.peek() == Some(';') { + self.advance(); + } + } + + self.depth -= 1; + Ok(YsonValue::List(items)) + } + + fn parse_map(&mut self) -> Result { + self.advance(); // skip '{' + self.depth += 1; + let map = self.parse_map_content('}')?; + self.depth -= 1; + Ok(YsonValue::Map(map)) + } + + fn parse_map_content(&mut self, terminator: char) -> Result, YsonValue>> { + let mut map = IndexMap::new(); + + loop { + self.skip_whitespace(); + if self.peek() == Some(terminator) { + self.advance(); + break; + } + + let key = self.parse_key()?; + self.skip_whitespace(); + + if self.peek() != Some('=') { + return Err(Error::UnexpectedToken { + expected: "'='", + found: format!("{:?}", self.peek()), + }); + } + self.advance(); + + let value = self.parse_value()?; + map.insert(key, value); + + self.skip_whitespace(); + if self.peek() == Some(';') { + self.advance(); + } + } + + Ok(map) + } + + fn parse_key(&mut self) -> Result> { + self.skip_whitespace(); + let c = self.peek().ok_or(Error::UnexpectedEof)?; + + if c == '"' { + self.advance(); + self.parse_string_content('"') + } else if c.is_ascii_alphabetic() || c == '_' { + let start = self.pos; + while self.pos < self.input.len() { + let c = self.input.as_bytes()[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'-' { + self.pos += 1; + } else { + break; + } + } + Ok(self.input[start..self.pos].as_bytes().to_vec()) + } else { + Err(Error::SyntaxError(format!("expected key, found: {}", c))) + } + } +} + +/// Parse text YSON +pub fn parse(input: &str) -> Result { + let mut reader = TextReader::new(input); + reader.parse_node() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_entity() { + assert_eq!(parse("#").unwrap(), YsonValue::Entity); + } + + #[test] + fn test_parse_booleans() { + assert_eq!(parse("%true").unwrap(), YsonValue::Boolean(true)); + assert_eq!(parse("%false").unwrap(), YsonValue::Boolean(false)); + } + + #[test] + fn test_parse_numbers() { + assert_eq!(parse("42").unwrap(), YsonValue::Int64(42)); + assert_eq!(parse("-1").unwrap(), YsonValue::Int64(-1)); + assert_eq!(parse("+100").unwrap(), YsonValue::Int64(100)); + assert_eq!(parse("42u").unwrap(), YsonValue::Uint64(42)); + assert_eq!(parse("3.14").unwrap(), YsonValue::Double(3.14)); + assert_eq!(parse("1e10").unwrap(), YsonValue::Double(1e10)); + assert_eq!(parse("1.5e-3").unwrap(), YsonValue::Double(1.5e-3)); + } + + #[test] + fn test_parse_special_floats() { + assert!(parse("%nan").unwrap().as_f64().unwrap().is_nan()); + assert_eq!(parse("%inf").unwrap(), YsonValue::Double(f64::INFINITY)); + assert_eq!(parse("%-inf").unwrap(), YsonValue::Double(f64::NEG_INFINITY)); + } + + #[test] + fn test_parse_strings() { + assert_eq!(parse("\"hello\"").unwrap().as_str(), Some("hello")); + assert_eq!(parse("\"hello world\"").unwrap().as_str(), Some("hello world")); + assert_eq!(parse("\"line1\\nline2\"").unwrap().as_str(), Some("line1\nline2")); + assert_eq!(parse("identifier").unwrap().as_str(), Some("identifier")); + assert_eq!(parse("foo_bar-123").unwrap().as_str(), Some("foo_bar-123")); + } + + #[test] + fn test_parse_list() { + let value = parse("[1; 2; 3]").unwrap(); + let list = value.as_list().unwrap(); + assert_eq!(list.len(), 3); + assert_eq!(list[0].as_i64(), Some(1)); + assert_eq!(list[1].as_i64(), Some(2)); + assert_eq!(list[2].as_i64(), Some(3)); + } + + #[test] + fn test_parse_empty_list() { + let value = parse("[]").unwrap(); + let list = value.as_list().unwrap(); + assert!(list.is_empty()); + } + + #[test] + fn test_parse_map() { + let value = parse("{a=1; b=2}").unwrap(); + let map = value.as_map().unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get(b"a".as_ref()).unwrap().as_i64(), Some(1)); + assert_eq!(map.get(b"b".as_ref()).unwrap().as_i64(), Some(2)); + } + + #[test] + fn test_parse_empty_map() { + let value = parse("{}").unwrap(); + let map = value.as_map().unwrap(); + assert!(map.is_empty()); + } + + #[test] + fn test_parse_nested() { + let value = parse("{list=[1;2;3]; nested={x=1}}").unwrap(); + let map = value.as_map().unwrap(); + assert_eq!(map.get(b"list".as_ref()).unwrap().as_list().unwrap().len(), 3); + let nested = map.get(b"nested".as_ref()).unwrap().as_map().unwrap(); + assert_eq!(nested.get(b"x".as_ref()).unwrap().as_i64(), Some(1)); + } + + #[test] + fn test_parse_with_whitespace() { + let value = parse(" { a = 1 ; b = 2 } ").unwrap(); + let map = value.as_map().unwrap(); + assert_eq!(map.len(), 2); + } + + #[test] + fn test_parse_attributes() { + // Attributes are parsed but discarded + let value = parse(" #").unwrap(); + assert_eq!(value, YsonValue::Entity); + } +} diff --git a/yt/rust/ytsaurus-yson/src/text/writer.rs b/yt/rust/ytsaurus-yson/src/text/writer.rs new file mode 100644 index 000000000000..601a68d2ad85 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/text/writer.rs @@ -0,0 +1,284 @@ +//! Text YSON writer. + +use crate::value::YsonValue; +use std::fmt::Write; + +/// Text YSON format style +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TextFormat { + /// Compact format (no unnecessary whitespace) + Compact, + /// Pretty format with indentation + Pretty, +} + +/// Text YSON writer +pub struct TextWriter { + buffer: String, + format: TextFormat, + indent: usize, + current_indent: usize, +} + +impl TextWriter { + /// Create a new text writer with compact format + pub fn new() -> Self { + Self { + buffer: String::new(), + format: TextFormat::Compact, + indent: 2, + current_indent: 0, + } + } + + /// Create a new text writer with pretty format + pub fn pretty() -> Self { + Self { + buffer: String::new(), + format: TextFormat::Pretty, + indent: 2, + current_indent: 0, + } + } + + /// Get the written string + pub fn into_string(self) -> String { + self.buffer + } + + /// Get a reference to the written string + pub fn as_str(&self) -> &str { + &self.buffer + } + + /// Clear the buffer + pub fn clear(&mut self) { + self.buffer.clear(); + self.current_indent = 0; + } + + /// Write a YSON value + pub fn write_value(&mut self, value: &YsonValue) { + match value { + YsonValue::Entity => self.buffer.push('#'), + YsonValue::Boolean(true) => self.buffer.push_str("%true"), + YsonValue::Boolean(false) => self.buffer.push_str("%false"), + YsonValue::Int64(n) => write!(self.buffer, "{}", n).unwrap(), + YsonValue::Uint64(n) => write!(self.buffer, "{}u", n).unwrap(), + YsonValue::Double(n) => self.write_double(*n), + YsonValue::String(bytes) => self.write_string(bytes), + YsonValue::List(items) => self.write_list(items), + YsonValue::Map(map) => self.write_map(map), + } + } + + fn write_double(&mut self, n: f64) { + if n.is_nan() { + self.buffer.push_str("%nan"); + } else if n.is_infinite() { + if n > 0.0 { + self.buffer.push_str("%inf"); + } else { + self.buffer.push_str("%-inf"); + } + } else { + write!(self.buffer, "{}", n).unwrap(); + } + } + + fn write_string(&mut self, bytes: &[u8]) { + if let Ok(s) = std::str::from_utf8(bytes) { + if is_identifier(s) { + self.buffer.push_str(s); + return; + } + } + self.write_quoted_string(bytes); + } + + fn write_quoted_string(&mut self, bytes: &[u8]) { + self.buffer.push('"'); + for &b in bytes { + match b { + b'"' => self.buffer.push_str("\\\""), + b'\\' => self.buffer.push_str("\\\\"), + b'\n' => self.buffer.push_str("\\n"), + b'\r' => self.buffer.push_str("\\r"), + b'\t' => self.buffer.push_str("\\t"), + 0x20..=0x7E => self.buffer.push(b as char), + _ => write!(self.buffer, "\\x{:02x}", b).unwrap(), + } + } + self.buffer.push('"'); + } + + fn write_list(&mut self, items: &[YsonValue]) { + self.buffer.push('['); + if self.format == TextFormat::Pretty && !items.is_empty() { + self.current_indent += self.indent; + } + for (i, item) in items.iter().enumerate() { + if i > 0 { + self.buffer.push(';'); + } + if self.format == TextFormat::Pretty { + self.buffer.push('\n'); + self.write_indent(); + } + self.write_value(item); + } + if self.format == TextFormat::Pretty && !items.is_empty() { + self.current_indent -= self.indent; + self.buffer.push('\n'); + self.write_indent(); + } + self.buffer.push(']'); + } + + fn write_map(&mut self, map: &indexmap::IndexMap, YsonValue>) { + self.buffer.push('{'); + if self.format == TextFormat::Pretty && !map.is_empty() { + self.current_indent += self.indent; + } + for (i, (key, value)) in map.iter().enumerate() { + if i > 0 { + self.buffer.push(';'); + } + if self.format == TextFormat::Pretty { + self.buffer.push('\n'); + self.write_indent(); + } + self.write_string(key); + self.buffer.push('='); + self.write_value(value); + } + if self.format == TextFormat::Pretty && !map.is_empty() { + self.current_indent -= self.indent; + self.buffer.push('\n'); + self.write_indent(); + } + self.buffer.push('}'); + } + + fn write_indent(&mut self) { + for _ in 0..self.current_indent { + self.buffer.push(' '); + } + } +} + +impl Default for TextWriter { + fn default() -> Self { + Self::new() + } +} + +/// Check if a string is a valid YSON identifier +fn is_identifier(s: &str) -> bool { + if s.is_empty() { + return false; + } + let mut chars = s.chars(); + let first = chars.next().unwrap(); + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.' || c == '-') +} + +/// Write a YSON value to text format +pub fn write(value: &YsonValue) -> String { + let mut writer = TextWriter::new(); + writer.write_value(value); + writer.into_string() +} + +/// Write a YSON value to pretty text format +pub fn write_pretty(value: &YsonValue) -> String { + let mut writer = TextWriter::pretty(); + writer.write_value(value); + writer.into_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use indexmap::IndexMap; + + #[test] + fn test_write_scalars() { + assert_eq!(write(&YsonValue::Entity), "#"); + assert_eq!(write(&YsonValue::Boolean(true)), "%true"); + assert_eq!(write(&YsonValue::Boolean(false)), "%false"); + assert_eq!(write(&YsonValue::Int64(42)), "42"); + assert_eq!(write(&YsonValue::Int64(-1)), "-1"); + assert_eq!(write(&YsonValue::Uint64(100)), "100u"); + assert_eq!(write(&YsonValue::Double(3.14)), "3.14"); + } + + #[test] + fn test_write_special_doubles() { + assert_eq!(write(&YsonValue::Double(f64::NAN)), "%nan"); + assert_eq!(write(&YsonValue::Double(f64::INFINITY)), "%inf"); + assert_eq!(write(&YsonValue::Double(f64::NEG_INFINITY)), "%-inf"); + } + + #[test] + fn test_write_strings() { + assert_eq!(write(&YsonValue::string("hello")), "hello"); + assert_eq!(write(&YsonValue::string("hello world")), "\"hello world\""); + assert_eq!(write(&YsonValue::string("line1\nline2")), "\"line1\\nline2\""); + assert_eq!(write(&YsonValue::string("")), "\"\""); + } + + #[test] + fn test_write_list() { + let list = YsonValue::List(vec![ + YsonValue::Int64(1), + YsonValue::Int64(2), + YsonValue::Int64(3), + ]); + assert_eq!(write(&list), "[1;2;3]"); + } + + #[test] + fn test_write_map() { + let mut map = IndexMap::new(); + map.insert(b"a".to_vec(), YsonValue::Int64(1)); + map.insert(b"b".to_vec(), YsonValue::Int64(2)); + let value = YsonValue::Map(map); + assert_eq!(write(&value), "{a=1;b=2}"); + } + + #[test] + fn test_write_pretty() { + let list = YsonValue::List(vec![ + YsonValue::Int64(1), + YsonValue::Int64(2), + ]); + let pretty = write_pretty(&list); + assert!(pretty.contains('\n')); + assert!(pretty.contains(" 1")); // indented + } + + #[test] + fn test_roundtrip() { + use crate::text::reader::parse; + + let test_values = vec![ + YsonValue::Entity, + YsonValue::Boolean(true), + YsonValue::Int64(42), + YsonValue::Uint64(100), + YsonValue::Double(3.14), + YsonValue::string("hello"), + YsonValue::List(vec![YsonValue::Int64(1), YsonValue::Int64(2)]), + ]; + + for value in test_values { + let text = write(&value); + let parsed = parse(&text).unwrap(); + assert_eq!(parsed, value, "roundtrip failed for {:?}", value); + } + } +} diff --git a/yt/rust/ytsaurus-yson/src/value.rs b/yt/rust/ytsaurus-yson/src/value.rs new file mode 100644 index 000000000000..2a8a433c72f6 --- /dev/null +++ b/yt/rust/ytsaurus-yson/src/value.rs @@ -0,0 +1,358 @@ +//! YSON value types. + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// YSON document type +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum YsonType { + /// Complete YSON node + Node, + /// Stream of list items (semicolon-separated) + ListFragment, + /// Stream of map key-value pairs + MapFragment, +} + +/// YSON value without attributes +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum YsonValue { + /// Entity (null) + Entity, + /// Boolean value + Boolean(bool), + /// Signed 64-bit integer + Int64(i64), + /// Unsigned 64-bit integer + Uint64(u64), + /// Double-precision floating point + Double(f64), + /// Binary string (may or may not be valid UTF-8) + String(Vec), + /// List of values + List(Vec), + /// Map from string keys to values + Map(IndexMap, YsonValue>), +} + +impl YsonValue { + /// Create a string value from a Rust string + pub fn string(s: impl Into) -> Self { + YsonValue::String(s.into().into_bytes()) + } + + /// Get as boolean if this is a boolean value + pub fn as_bool(&self) -> Option { + match self { + YsonValue::Boolean(b) => Some(*b), + _ => None, + } + } + + /// Get as i64 if this is an integer value + pub fn as_i64(&self) -> Option { + match self { + YsonValue::Int64(n) => Some(*n), + YsonValue::Uint64(n) => i64::try_from(*n).ok(), + _ => None, + } + } + + /// Get as u64 if this is an unsigned integer value + pub fn as_u64(&self) -> Option { + match self { + YsonValue::Uint64(n) => Some(*n), + YsonValue::Int64(n) => u64::try_from(*n).ok(), + _ => None, + } + } + + /// Get as f64 if this is a double value + pub fn as_f64(&self) -> Option { + match self { + YsonValue::Double(n) => Some(*n), + YsonValue::Int64(n) => Some(*n as f64), + YsonValue::Uint64(n) => Some(*n as f64), + _ => None, + } + } + + /// Get as string slice if this is a string value with valid UTF-8 + pub fn as_str(&self) -> Option<&str> { + match self { + YsonValue::String(bytes) => std::str::from_utf8(bytes).ok(), + _ => None, + } + } + + /// Get as byte slice if this is a string value + pub fn as_bytes(&self) -> Option<&[u8]> { + match self { + YsonValue::String(bytes) => Some(bytes), + _ => None, + } + } + + /// Get as list if this is a list value + pub fn as_list(&self) -> Option<&Vec> { + match self { + YsonValue::List(list) => Some(list), + _ => None, + } + } + + /// Get as map if this is a map value + pub fn as_map(&self) -> Option<&IndexMap, YsonValue>> { + match self { + YsonValue::Map(map) => Some(map), + _ => None, + } + } + + /// Check if this is an entity (null) + pub fn is_entity(&self) -> bool { + matches!(self, YsonValue::Entity) + } + + /// Get a value from a map by string key + pub fn get(&self, key: &str) -> Option<&YsonValue> { + match self { + YsonValue::Map(map) => map.get(key.as_bytes()), + _ => None, + } + } + + /// Get a value from a list by index + pub fn get_index(&self, index: usize) -> Option<&YsonValue> { + match self { + YsonValue::List(list) => list.get(index), + _ => None, + } + } +} + +impl Default for YsonValue { + fn default() -> Self { + YsonValue::Entity + } +} + +impl fmt::Display for YsonValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + YsonValue::Entity => write!(f, "#"), + YsonValue::Boolean(true) => write!(f, "%true"), + YsonValue::Boolean(false) => write!(f, "%false"), + YsonValue::Int64(n) => write!(f, "{}", n), + YsonValue::Uint64(n) => write!(f, "{}u", n), + YsonValue::Double(n) => { + if n.is_nan() { + write!(f, "%nan") + } else if n.is_infinite() { + if *n > 0.0 { + write!(f, "%inf") + } else { + write!(f, "%-inf") + } + } else { + write!(f, "{}", n) + } + } + YsonValue::String(bytes) => { + if let Ok(s) = std::str::from_utf8(bytes) { + if is_identifier(s) { + write!(f, "{}", s) + } else { + write!(f, "\"{}\"", escape_string(s)) + } + } else { + write!(f, "\"{}\"", escape_bytes(bytes)) + } + } + YsonValue::List(list) => { + write!(f, "[")?; + for (i, item) in list.iter().enumerate() { + if i > 0 { + write!(f, ";")?; + } + write!(f, "{}", item)?; + } + write!(f, "]") + } + YsonValue::Map(map) => { + write!(f, "{{")?; + for (i, (key, value)) in map.iter().enumerate() { + if i > 0 { + write!(f, ";")?; + } + if let Ok(s) = std::str::from_utf8(key) { + if is_identifier(s) { + write!(f, "{}={}", s, value)?; + } else { + write!(f, "\"{}\"={}", escape_string(s), value)?; + } + } else { + write!(f, "\"{}\"={}", escape_bytes(key), value)?; + } + } + write!(f, "}}") + } + } + } +} + +/// Check if a string is a valid YSON identifier (no quoting needed) +fn is_identifier(s: &str) -> bool { + if s.is_empty() { + return false; + } + let mut chars = s.chars(); + let first = chars.next().unwrap(); + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.' || c == '-') +} + +/// Escape a string for YSON text format +fn escape_string(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '"' => result.push_str("\\\""), + '\\' => result.push_str("\\\\"), + '\n' => result.push_str("\\n"), + '\r' => result.push_str("\\r"), + '\t' => result.push_str("\\t"), + c if c.is_ascii_control() => { + result.push_str(&format!("\\x{:02x}", c as u8)); + } + c => result.push(c), + } + } + result +} + +/// Escape bytes for YSON text format +fn escape_bytes(bytes: &[u8]) -> String { + let mut result = String::with_capacity(bytes.len() * 2); + for &b in bytes { + if b.is_ascii_graphic() || b == b' ' { + if b == b'"' { + result.push_str("\\\""); + } else if b == b'\\' { + result.push_str("\\\\"); + } else { + result.push(b as char); + } + } else { + result.push_str(&format!("\\x{:02x}", b)); + } + } + result +} + +// Implement From traits for convenient construction +impl From for YsonValue { + fn from(v: bool) -> Self { + YsonValue::Boolean(v) + } +} + +impl From for YsonValue { + fn from(v: i64) -> Self { + YsonValue::Int64(v) + } +} + +impl From for YsonValue { + fn from(v: i32) -> Self { + YsonValue::Int64(v as i64) + } +} + +impl From for YsonValue { + fn from(v: u64) -> Self { + YsonValue::Uint64(v) + } +} + +impl From for YsonValue { + fn from(v: u32) -> Self { + YsonValue::Uint64(v as u64) + } +} + +impl From for YsonValue { + fn from(v: f64) -> Self { + YsonValue::Double(v) + } +} + +impl From<&str> for YsonValue { + fn from(v: &str) -> Self { + YsonValue::String(v.as_bytes().to_vec()) + } +} + +impl From for YsonValue { + fn from(v: String) -> Self { + YsonValue::String(v.into_bytes()) + } +} + +impl From> for YsonValue { + fn from(v: Vec) -> Self { + YsonValue::String(v) + } +} + +impl> From> for YsonValue { + fn from(v: Vec) -> Self { + YsonValue::List(v.into_iter().map(Into::into).collect()) + } +} + +impl From<()> for YsonValue { + fn from(_: ()) -> Self { + YsonValue::Entity + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_display_scalars() { + assert_eq!(format!("{}", YsonValue::Entity), "#"); + assert_eq!(format!("{}", YsonValue::Boolean(true)), "%true"); + assert_eq!(format!("{}", YsonValue::Boolean(false)), "%false"); + assert_eq!(format!("{}", YsonValue::Int64(42)), "42"); + assert_eq!(format!("{}", YsonValue::Int64(-1)), "-1"); + assert_eq!(format!("{}", YsonValue::Uint64(100)), "100u"); + assert_eq!(format!("{}", YsonValue::string("hello")), "hello"); + assert_eq!(format!("{}", YsonValue::string("hello world")), "\"hello world\""); + } + + #[test] + fn test_display_list() { + let list = YsonValue::List(vec![ + YsonValue::Int64(1), + YsonValue::Int64(2), + YsonValue::Int64(3), + ]); + assert_eq!(format!("{}", list), "[1;2;3]"); + } + + #[test] + fn test_display_map() { + let mut map = IndexMap::new(); + map.insert(b"a".to_vec(), YsonValue::Int64(1)); + map.insert(b"b".to_vec(), YsonValue::string("hello")); + let value = YsonValue::Map(map); + assert_eq!(format!("{}", value), "{a=1;b=hello}"); + } +}