From b885373b4c6bed7602b967417dc890d8c5713798 Mon Sep 17 00:00:00 2001 From: amigin Date: Sun, 21 Apr 2024 23:38:49 +0300 Subject: [PATCH 01/89] Zip Fix --- my-no-sql-tcp-shared/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index ec134ab..2631162 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -12,4 +12,4 @@ rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-ext tokio = { version = "*", features = ["full"] } tokio-util = "*" async-trait = "*" -zip = { version = "*", default-features = false, features = ["deflate"] } +zip = { version = "0", default-features = false, features = ["deflate"] } From 78ce7068e9eb4deaacfde3ccc21caed661250ec3 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 26 Apr 2024 13:20:12 +0300 Subject: [PATCH 02/89] Now we can deserialize Options which are not in a list yet --- .../src/my_no_sql_entity.rs | 4 +- .../src/my_no_sql_data_writer/execution.rs | 11 +- my-no-sql-macros/src/entity_utils.rs | 6 +- my-no-sql-macros/src/enum_model/generate.rs | 1 + .../src/enum_of_my_no_sql_entity/generate.rs | 11 +- .../src/data_reader_entities_set.rs | 190 ++++++++++++++++++ my-no-sql-tcp-reader/src/lib.rs | 6 +- .../src/subscribers/callback_triggers.rs | 5 +- .../subscribers/my_no_sql_data_reader_data.rs | 143 ++----------- .../subscribers/my_no_sql_data_reader_tcp.rs | 12 +- my-no-sql-tests/Cargo.toml | 2 + my-no-sql-tests/src/lib.rs | 5 + my-no-sql-tests/src/macros_tests/enum_test.rs | 4 +- my-no-sql-tests/src/test_my_no_sql_entity.rs | 3 + .../src/test_new_enum_cases_added/mod.rs | 1 + .../test_enum_with_added_new_type.rs | 50 +++++ 16 files changed, 302 insertions(+), 152 deletions(-) create mode 100644 my-no-sql-tcp-reader/src/data_reader_entities_set.rs create mode 100644 my-no-sql-tests/src/test_my_no_sql_entity.rs create mode 100644 my-no-sql-tests/src/test_new_enum_cases_added/mod.rs create mode 100644 my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs diff --git a/my-no-sql-abstractions/src/my_no_sql_entity.rs b/my-no-sql-abstractions/src/my_no_sql_entity.rs index ac4906c..cb76ee2 100644 --- a/my-no-sql-abstractions/src/my_no_sql_entity.rs +++ b/my-no-sql-abstractions/src/my_no_sql_entity.rs @@ -5,9 +5,9 @@ pub trait MyNoSqlEntity { fn get_time_stamp(&self) -> i64; } -pub trait MyNoSqlEntitySerializer { +pub trait MyNoSqlEntitySerializer: Sized { fn serialize_entity(&self) -> Vec; - fn deserialize_entity(src: &[u8]) -> Self; + fn deserialize_entity(src: &[u8]) -> Option; } pub trait GetMyNoSqlEntity { diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 9aae94a..bdfbab4 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -169,7 +169,7 @@ pub async fn get_entity( for itm in JsonArrayIterator::new(src) { let itm = itm.unwrap(); - result.push(TEntity::deserialize_entity(itm)); + result.push(TEntity::deserialize_entity(itm).unwrap()); } Ok(result) } @@ -589,8 +589,9 @@ mod tests { my_no_sql_core::entity_serializer::serialize(self) } - fn deserialize_entity(src: &[u8]) -> Self { - my_no_sql_core::entity_serializer::deserialize(src) + fn deserialize_entity(src: &[u8]) -> Option { + let result: Self = my_no_sql_core::entity_serializer::deserialize(src); + Some(result) } } diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index afa6437..2da2e6b 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -109,8 +109,10 @@ pub fn get_fn_standard_serialize_deserialize() -> proc_macro2::TokenStream { my_no_sql_sdk::core::entity_serializer::serialize(self) } - fn deserialize_entity(src: &[u8]) -> Self { - my_no_sql_sdk::core::entity_serializer::deserialize(src) + + fn deserialize_entity(src: &[u8]) -> Option { + let result: Self = my_no_sql_sdk::core::entity_serializer::deserialize(src); + Some(result) } } } diff --git a/my-no-sql-macros/src/enum_model/generate.rs b/my-no-sql-macros/src/enum_model/generate.rs index 2a370aa..4e3f566 100644 --- a/my-no-sql-macros/src/enum_model/generate.rs +++ b/my-no-sql-macros/src/enum_model/generate.rs @@ -51,6 +51,7 @@ pub fn generate( const TABLE_NAME: &'static str = ""; + fn get_partition_key(&self) -> &str { Self::PARTITION_KEY } diff --git a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs index d730fa1..098477e 100644 --- a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs @@ -50,7 +50,6 @@ pub fn generate( const TABLE_NAME: &'static str = #table_name; - fn get_partition_key(&self) -> &str { use my_no_sql_sdk::abstractions::MyNoSqlEntity; match self { @@ -86,7 +85,7 @@ pub fn generate( my_no_sql_sdk::core::entity_serializer::inject_partition_key_and_row_key(result, self.get_partition_key(), row_key) } - fn deserialize_entity(src: &[u8]) -> Self { + fn deserialize_entity(src: &[u8]) -> Option { #deserialize_cases } } @@ -178,11 +177,13 @@ fn get_deserialize_cases(enum_cases: &[EnumCase]) -> Result Result { + entities: Option>>>, + table_name: &'static str, +} + +impl DataReaderEntitiesSet { + pub fn new(table_name: &'static str) -> Self { + Self { + entities: None, + table_name, + } + } + + pub fn is_initialized(&self) -> bool { + self.entities.is_some() + } + + pub fn as_ref(&self) -> Option<&BTreeMap>>> { + self.entities.as_ref() + } + + fn init_and_get_table( + &mut self, + ) -> &mut BTreeMap>> { + if self.entities.is_none() { + println!("MyNoSqlTcpReader table {} is initialized", self.table_name); + self.entities = Some(BTreeMap::new()); + return self.entities.as_mut().unwrap(); + } + + return self.entities.as_mut().unwrap(); + } + + pub fn init_table<'s>( + &'s mut self, + data: BTreeMap>, + ) -> InitTableResult<'s, TMyNoSqlEntity> { + let mut new_table: BTreeMap>> = + BTreeMap::new(); + + for (partition_key, src_entities_by_partition) in data { + new_table.insert(partition_key.to_string(), BTreeMap::new()); + + let by_partition = new_table.get_mut(partition_key.as_str()).unwrap(); + + for entity in src_entities_by_partition { + let entity = Arc::new(entity); + by_partition.insert(entity.get_row_key().to_string(), entity); + } + } + + let table_before = self.entities.replace(new_table); + + InitTableResult { + table_now: self.entities.as_ref().unwrap(), + table_before, + } + } + + pub fn init_partition<'s>( + &'s mut self, + partition_key: &str, + src_entities: BTreeMap>, + ) -> InitPartitionResult<'s, TMyNoSqlEntity> { + let entities = self.init_and_get_table(); + + let mut new_partition = BTreeMap::new(); + + let before_partition = entities.remove(partition_key); + + for (row_key, entities) in src_entities { + for entity in entities { + new_partition.insert(row_key.clone(), Arc::new(entity)); + } + } + + entities.insert(partition_key.to_string(), new_partition); + + InitPartitionResult { + partition_before: entities.get(partition_key).unwrap(), + partition_now: before_partition, + } + } + + pub fn update_rows( + &mut self, + src_data: BTreeMap>, + callbacks: &Option>>, + ) { + let entities = self.init_and_get_table(); + + for (partition_key, src_entities) in src_data { + let mut updates = if callbacks.is_some() { + Some(Vec::new()) + } else { + None + }; + + if !entities.contains_key(partition_key.as_str()) { + entities.insert(partition_key.to_string(), BTreeMap::new()); + } + + let by_partition = entities.get_mut(partition_key.as_str()).unwrap(); + + for entity in src_entities { + let entity = Arc::new(entity); + if let Some(updates) = updates.as_mut() { + updates.push(entity.clone()); + } + by_partition.insert(entity.get_row_key().to_string(), entity); + } + + if let Some(callbacks) = callbacks { + if let Some(updates) = updates { + if updates.len() > 0 { + callbacks.inserted_or_replaced(partition_key.as_str(), updates); + } + } + } + } + } + + pub fn delete_rows( + &mut self, + rows_to_delete: Vec, + callbacks: &Option>>, + ) { + let mut deleted_rows = if callbacks.is_some() { + Some(BTreeMap::new()) + } else { + None + }; + + let entities = self.init_and_get_table(); + + for row_to_delete in &rows_to_delete { + let mut delete_partition = false; + if let Some(partition) = entities.get_mut(row_to_delete.partition_key.as_str()) { + if partition.remove(row_to_delete.row_key.as_str()).is_some() { + if let Some(deleted_rows) = deleted_rows.as_mut() { + if !deleted_rows.contains_key(row_to_delete.partition_key.as_str()) { + deleted_rows + .insert(row_to_delete.partition_key.to_string(), Vec::new()); + } + + deleted_rows + .get_mut(row_to_delete.partition_key.as_str()) + .unwrap() + .push( + partition + .get(row_to_delete.row_key.as_str()) + .unwrap() + .clone(), + ); + } + } + + delete_partition = partition.len() == 0; + } + + if delete_partition { + entities.remove(row_to_delete.partition_key.as_str()); + } + } + + if let Some(callbacks) = callbacks.as_ref() { + if let Some(partitions) = deleted_rows { + for (partition_key, rows) in partitions { + callbacks.deleted(partition_key.as_str(), rows); + } + } + } + } +} + +pub struct InitTableResult<'s, TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static> { + pub table_now: &'s BTreeMap>>, + pub table_before: Option>>>, +} + +pub struct InitPartitionResult<'s, TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static> { + pub partition_before: &'s BTreeMap>, + pub partition_now: Option>>, +} diff --git a/my-no-sql-tcp-reader/src/lib.rs b/my-no-sql-tcp-reader/src/lib.rs index 1776664..60f97fc 100644 --- a/my-no-sql-tcp-reader/src/lib.rs +++ b/my-no-sql-tcp-reader/src/lib.rs @@ -1,11 +1,15 @@ +mod data_reader_entities_set; mod my_no_sql_tcp_connection; mod settings; mod subscribers; mod tcp_events; +pub use data_reader_entities_set::*; pub use my_no_sql_tcp_connection::MyNoSqlTcpConnection; pub use settings::*; -pub use subscribers::{MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderData, MyNoSqlDataReaderTcp, MyNoSqlDataReader}; +pub use subscribers::{ + MyNoSqlDataReader, MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderData, MyNoSqlDataReaderTcp, +}; #[cfg(feature = "mocks")] pub use subscribers::MyNoSqlDataReaderMock; diff --git a/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs b/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs index b62448b..e7ad1fd 100644 --- a/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs +++ b/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs @@ -246,8 +246,9 @@ mod tests { my_no_sql_core::entity_serializer::serialize(self) } - fn deserialize_entity(src: &[u8]) -> Self { - my_no_sql_core::entity_serializer::deserialize(src) + fn deserialize_entity(src: &[u8]) -> Option { + let result: Self = my_no_sql_core::entity_serializer::deserialize(src); + Some(result) } } diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs index 61471dc..0c3ebfb 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs @@ -3,11 +3,12 @@ use std::{collections::BTreeMap, sync::Arc}; use my_no_sql_abstractions::MyNoSqlEntity; use rust_extensions::ApplicationStates; +use crate::DataReaderEntitiesSet; + use super::{MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderCallBacksPusher}; pub struct MyNoSqlDataReaderData { - table_name: &'static str, - entities: Option>>>, + entities: DataReaderEntitiesSet, callbacks: Option>>, app_states: Arc, } @@ -21,8 +22,7 @@ where app_states: Arc, ) -> Self { Self { - table_name, - entities: None, + entities: DataReaderEntitiesSet::new(table_name), callbacks: None, app_states, } @@ -39,38 +39,15 @@ where self.callbacks = Some(Arc::new(pusher)); } - fn get_init_table(&mut self) -> &mut BTreeMap>> { - if self.entities.is_none() { - println!("Initialized data for table {}", self.table_name); - self.entities = Some(BTreeMap::new()); - return self.entities.as_mut().unwrap(); - } - - return self.entities.as_mut().unwrap(); - } pub async fn init_table(&mut self, data: BTreeMap>) { - let mut new_table: BTreeMap>> = - BTreeMap::new(); - - for (partition_key, src_entities_by_partition) in data { - new_table.insert(partition_key.to_string(), BTreeMap::new()); - - let by_partition = new_table.get_mut(partition_key.as_str()).unwrap(); - - for entity in src_entities_by_partition { - let entity = Arc::new(entity); - by_partition.insert(entity.get_row_key().to_string(), entity); - } - } - - let before = self.entities.replace(new_table); + let init_table_result = self.entities.init_table(data); if let Some(callbacks) = self.callbacks.as_ref() { super::callback_triggers::trigger_table_difference( callbacks.as_ref(), - before, - self.entities.as_ref().unwrap(), + init_table_result.table_before, + init_table_result.table_now, ) .await; } @@ -81,117 +58,27 @@ where partition_key: &str, src_entities: BTreeMap>, ) { - let callbacks = self.callbacks.clone(); - - let entities = self.get_init_table(); - - let mut new_partition = BTreeMap::new(); - - let before_partition = entities.remove(partition_key); + //let callbacks = self.callbacks.clone(); - for (row_key, entities) in src_entities { - for entity in entities { - new_partition.insert(row_key.clone(), Arc::new(entity)); - } - } + let init_partition_result = self.entities.init_partition(partition_key, src_entities); - entities.insert(partition_key.to_string(), new_partition); - - if let Some(callbacks) = callbacks { + if let Some(callbacks) = self.callbacks.as_ref() { super::callback_triggers::trigger_partition_difference( callbacks.as_ref(), partition_key, - before_partition, - entities.get(partition_key).unwrap(), + init_partition_result.partition_now, + init_partition_result.partition_before, ) .await; } } pub fn update_rows(&mut self, src_data: BTreeMap>) { - let callbacks = self.callbacks.clone(); - - let entities = self.get_init_table(); - - for (partition_key, src_entities) in src_data { - let mut updates = if callbacks.is_some() { - Some(Vec::new()) - } else { - None - }; - - if !entities.contains_key(partition_key.as_str()) { - entities.insert(partition_key.to_string(), BTreeMap::new()); - } - - let by_partition = entities.get_mut(partition_key.as_str()).unwrap(); - - for entity in src_entities { - let entity = Arc::new(entity); - if let Some(updates) = updates.as_mut() { - updates.push(entity.clone()); - } - by_partition.insert(entity.get_row_key().to_string(), entity); - } - - if let Some(callbacks) = callbacks.as_ref() { - if let Some(updates) = updates { - if updates.len() > 0 { - callbacks.inserted_or_replaced(partition_key.as_str(), updates); - } - } - } - } + self.entities.update_rows(src_data, &self.callbacks); } pub fn delete_rows(&mut self, rows_to_delete: Vec) { - let callbacks = self.callbacks.clone(); - - let mut deleted_rows = if callbacks.is_some() { - Some(BTreeMap::new()) - } else { - None - }; - - let entities = self.get_init_table(); - - for row_to_delete in &rows_to_delete { - let mut delete_partition = false; - if let Some(partition) = entities.get_mut(row_to_delete.partition_key.as_str()) { - if partition.remove(row_to_delete.row_key.as_str()).is_some() { - if let Some(deleted_rows) = deleted_rows.as_mut() { - if !deleted_rows.contains_key(row_to_delete.partition_key.as_str()) { - deleted_rows - .insert(row_to_delete.partition_key.to_string(), Vec::new()); - } - - deleted_rows - .get_mut(row_to_delete.partition_key.as_str()) - .unwrap() - .push( - partition - .get(row_to_delete.row_key.as_str()) - .unwrap() - .clone(), - ); - } - } - - delete_partition = partition.len() == 0; - } - - if delete_partition { - entities.remove(row_to_delete.partition_key.as_str()); - } - } - - if let Some(callbacks) = callbacks.as_ref() { - if let Some(partitions) = deleted_rows { - for (partition_key, rows) in partitions { - callbacks.deleted(partition_key.as_str(), rows); - } - } - } + self.entities.delete_rows(rows_to_delete, &self.callbacks); } pub fn get_table_snapshot( @@ -316,6 +203,6 @@ where } pub async fn has_entities_at_all(&self) -> bool { - self.entities.is_some() + self.entities.is_initialized() } } diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 43c5761..a913d3f 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -124,12 +124,14 @@ where let el = TMyNoSqlEntity::deserialize_entity(db_entity_data); - let partition_key = el.get_partition_key(); - if !result.contains_key(partition_key) { - result.insert(partition_key.to_string(), Vec::new()); - } + if let Some(el) = el { + let partition_key = el.get_partition_key(); + if !result.contains_key(partition_key) { + result.insert(partition_key.to_string(), Vec::new()); + } - result.get_mut(partition_key).unwrap().push(el); + result.get_mut(partition_key).unwrap().push(el); + } } result diff --git a/my-no-sql-tests/Cargo.toml b/my-no-sql-tests/Cargo.toml index 9b3b80c..b708996 100644 --- a/my-no-sql-tests/Cargo.toml +++ b/my-no-sql-tests/Cargo.toml @@ -12,3 +12,5 @@ async-trait = "*" my-no-sql-macros = { path = "../my-no-sql-macros" } serde = { version = "*", features = ["derive"] } my-no-sql-sdk = { path = "../my-no-sql-sdk" } + +my-no-sql-tcp-reader = { path = "../my-no-sql-tcp-reader" } diff --git a/my-no-sql-tests/src/lib.rs b/my-no-sql-tests/src/lib.rs index 8e24695..737d8ac 100644 --- a/my-no-sql-tests/src/lib.rs +++ b/my-no-sql-tests/src/lib.rs @@ -1,2 +1,7 @@ #[cfg(test)] mod macros_tests; +#[cfg(test)] +mod test_new_enum_cases_added; + +#[cfg(test)] +mod test_my_no_sql_entity; diff --git a/my-no-sql-tests/src/macros_tests/enum_test.rs b/my-no-sql-tests/src/macros_tests/enum_test.rs index 747d377..8f268e5 100644 --- a/my-no-sql-tests/src/macros_tests/enum_test.rs +++ b/my-no-sql-tests/src/macros_tests/enum_test.rs @@ -33,7 +33,7 @@ fn test_serialize_deserialize_case_1() { let vec = entity.serialize_entity(); - let dest = MyNoSqlEnumEntityTest::deserialize_entity(&vec); + let dest = MyNoSqlEnumEntityTest::deserialize_entity(&vec).unwrap(); let model = dest.unwrap_case1(); @@ -53,7 +53,7 @@ fn test_serialize_deserialize_case_2() { let vec = entity.serialize_entity(); - let dest = MyNoSqlEnumEntityTest::deserialize_entity(&vec); + let dest = MyNoSqlEnumEntityTest::deserialize_entity(&vec).unwrap(); let model = dest.unwrap_case2(); diff --git a/my-no-sql-tests/src/test_my_no_sql_entity.rs b/my-no-sql-tests/src/test_my_no_sql_entity.rs new file mode 100644 index 0000000..def473e --- /dev/null +++ b/my-no-sql-tests/src/test_my_no_sql_entity.rs @@ -0,0 +1,3 @@ +pub struct MyNoSqlEntity { + pub my_field: String, +} diff --git a/my-no-sql-tests/src/test_new_enum_cases_added/mod.rs b/my-no-sql-tests/src/test_new_enum_cases_added/mod.rs new file mode 100644 index 0000000..a1a46e3 --- /dev/null +++ b/my-no-sql-tests/src/test_new_enum_cases_added/mod.rs @@ -0,0 +1 @@ +mod test_enum_with_added_new_type; diff --git a/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs b/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs new file mode 100644 index 0000000..fe18c1b --- /dev/null +++ b/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs @@ -0,0 +1,50 @@ +use my_no_sql_macros::*; +use serde::*; + +#[enum_of_my_no_sql_entity(table_name:"Test", generate_unwraps)] +pub enum MyNoSqlEnumEntityTestVer1 { + Case1(Struct1), +} + +#[enum_of_my_no_sql_entity(table_name:"Test", generate_unwraps)] +pub enum MyNoSqlEnumEntityTestVer2 { + Case1(Struct1), + Case2(Struct2), +} + +#[enum_model(partition_key:"pk1", row_key: "rk1")] +#[derive(Serialize, Deserialize, Clone)] +pub struct Struct1 { + pub field1: String, + pub field2: i32, +} + +#[enum_model(partition_key:"pk2", row_key: "rk2")] +#[derive(Serialize, Deserialize, Clone)] +pub struct Struct2 { + pub field3: String, + pub field4: i32, +} + +#[cfg(test)] +mod tests { + + use my_no_sql_sdk::abstractions::MyNoSqlEntitySerializer; + + use super::*; + + #[test] + fn test() { + let model_ver2 = MyNoSqlEnumEntityTestVer2::Case2(Struct2 { + time_stamp: "".to_string(), + field3: "field3".to_string(), + field4: 4, + }); + + let result = model_ver2.serialize_entity(); + + let model_ver1 = MyNoSqlEnumEntityTestVer1::deserialize_entity(result.as_slice()); + + assert!(model_ver1.is_none()) + } +} From dbd218b445f323ecdef40fc3cf3287647b29f2e8 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 9 Jul 2024 18:51:07 +0300 Subject: [PATCH 03/89] Last version of flurl --- my-no-sql-data-writer/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 03db819..b17a05b 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -13,7 +13,7 @@ my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ "with-tokio", ] } -flurl = { tag = "0.5.0", git = "https://github.com/MyJetTools/fl-url" } +flurl = { tag = "0.5.2", git = "https://github.com/MyJetTools/fl-url" } my-json = { tag = "0.2.2", git = "https://github.com/MyJetTools/my-json.git" } From 8e3f42ec9a6cb55e6b405bf49420ee489e19e66f Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 9 Jul 2024 18:59:51 +0300 Subject: [PATCH 04/89] 0.3.1 --- my-no-sql-abstractions/Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 2 +- my-no-sql-data-writer/Cargo.toml | 2 +- my-no-sql-macros/Cargo.toml | 2 +- my-no-sql-sdk/Cargo.toml | 2 +- my-no-sql-tcp-reader/Cargo.toml | 2 +- my-no-sql-tcp-shared/Cargo.toml | 2 +- my-no-sql-tests/Cargo.toml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/my-no-sql-abstractions/Cargo.toml b/my-no-sql-abstractions/Cargo.toml index cdd11a0..28cd9bb 100644 --- a/my-no-sql-abstractions/Cargo.toml +++ b/my-no-sql-abstractions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-abstractions" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index d2c6e90..a08a629 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-core" -version = "0.3.0" +version = "0.3.1" edition = "2021" [features] diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index b17a05b..b4f880b 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-data-writer" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-macros/Cargo.toml b/my-no-sql-macros/Cargo.toml index 767db82..5105e18 100644 --- a/my-no-sql-macros/Cargo.toml +++ b/my-no-sql-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-macros" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index eee19e0..b8c078e 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-sdk" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index 4103860..59be187 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-reader" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 2631162..4de2b0f 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-shared" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tests/Cargo.toml b/my-no-sql-tests/Cargo.toml index b708996..0206965 100644 --- a/my-no-sql-tests/Cargo.toml +++ b/my-no-sql-tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tests" -version = "0.3.0" +version = "0.3.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From f95cc03148bea4625611bcdcf0c17a310cd766ac Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 9 Jul 2024 19:28:03 +0300 Subject: [PATCH 05/89] Last version of my-json --- my-no-sql-core/Cargo.toml | 2 +- .../src/db_json_entity/db_json_entity.rs | 177 ++++++++++++++---- .../db_json_entity_with_content.rs | 7 +- .../db_row_content_compiler.rs | 9 +- .../db_json_entity/json_key_value_position.rs | 12 +- my-no-sql-core/src/entity_serializer.rs | 12 +- my-no-sql-data-writer/Cargo.toml | 2 +- .../src/my_no_sql_data_writer/execution.rs | 21 ++- my-no-sql-macros/src/entity_utils.rs | 2 +- .../src/enum_of_my_no_sql_entity/generate.rs | 2 +- my-no-sql-tcp-reader/Cargo.toml | 4 +- .../src/subscribers/callback_triggers.rs | 2 +- .../subscribers/my_no_sql_data_reader_tcp.rs | 39 +++- 13 files changed, 227 insertions(+), 64 deletions(-) diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index a08a629..2082449 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -13,7 +13,7 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -my-json = { tag = "0.2.2", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.2.4", git = "https://github.com/MyJetTools/my-json.git" } rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } serde_json = { version = "*" } diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity.rs b/my-no-sql-core/src/db_json_entity/db_json_entity.rs index a4f6e4a..0a383c2 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity.rs @@ -1,6 +1,7 @@ use crate::db::DbRow; -use my_json::json_reader::array_parser::ArrayToJsonObjectsSplitter; +use my_json::json_reader::array_iterator::JsonArrayIterator; +use rust_extensions::array_of_bytes_iterator::SliceIterator; use rust_extensions::date_time::DateTimeAsMicroseconds; use std::sync::Arc; @@ -22,7 +23,13 @@ pub struct DbJsonEntity { } impl DbJsonEntity { - pub fn new(raw: &[u8]) -> Result { + pub fn from_slice(src: &[u8]) -> Result { + let slice_iterator = SliceIterator::new(src); + Self::new(JsonFirstLineReader::new(slice_iterator)) + } + pub fn new( + mut json_first_line_reader: JsonFirstLineReader, + ) -> Result { let mut partition_key = None; let mut row_key = None; let mut expires = None; @@ -30,10 +37,10 @@ impl DbJsonEntity { let mut expires_value = None; - for line in JsonFirstLineReader::new(raw) { + while let Some(line) = json_first_line_reader.get_next() { let line = line?; - let name = line.get_name()?; + let name = line.name.as_unescaped_name(&json_first_line_reader)?; match name { super::consts::PARTITION_KEY => { partition_key = Some(JsonKeyValuePosition::new(&line)); @@ -43,7 +50,7 @@ impl DbJsonEntity { row_key = Some(JsonKeyValuePosition::new(&line)); } super::consts::EXPIRES => { - expires_value = line.get_value()?.as_date_time(); + expires_value = line.value.as_date_time(&json_first_line_reader); expires = Some(JsonKeyValuePosition::new(&line)) } super::consts::TIME_STAMP => { @@ -59,6 +66,7 @@ impl DbJsonEntity { } } } + let raw = json_first_line_reader.get_src_slice(); if partition_key.is_none() { return Err(DbEntityParseFail::FieldPartitionKeyIsRequired); @@ -99,7 +107,8 @@ impl DbJsonEntity { raw: &'s [u8], time_stamp_to_inject: &'s JsonTimeStamp, ) -> Result, DbEntityParseFail> { - let entity = Self::new(raw)?; + let slice_iterator = SliceIterator::new(raw); + let entity = Self::new(JsonFirstLineReader::new(slice_iterator))?; return Ok(DbJsonEntityWithContent::new( raw, @@ -108,33 +117,38 @@ impl DbJsonEntity { )); } - pub fn parse_into_db_row(src: &[u8], now: &JsonTimeStamp) -> Result { + pub fn parse_into_db_row( + mut json_first_line_reader: JsonFirstLineReader, + now: &JsonTimeStamp, + ) -> Result { let mut partition_key = None; let mut row_key = None; let mut expires = None; let mut time_stamp = None; let mut expires_value = None; - let mut raw = DbRowContentCompiler::new(src.len()); + let mut raw = DbRowContentCompiler::new(json_first_line_reader.get_src_slice().len()); - for line in JsonFirstLineReader::new(&src) { + while let Some(line) = json_first_line_reader.get_next() { let line = line?; - let name = line.get_name()?; + let line_ref = line.as_ref(&json_first_line_reader); + + let name = line_ref.name.as_unescaped_name().unwrap(); match name { super::consts::PARTITION_KEY => { - partition_key = Some(raw.append(src, &line)); + partition_key = Some(raw.append(line_ref)); } super::consts::ROW_KEY => { - row_key = Some(raw.append(src, &line)); + row_key = Some(raw.append(line_ref)); time_stamp = raw .append_str_value(super::consts::TIME_STAMP, now.as_str()) .into(); } super::consts::EXPIRES => { - expires_value = line.get_value()?.as_date_time(); - expires = Some(raw.append(src, &line)); + expires_value = line.value.as_date_time(&json_first_line_reader); + expires = Some(raw.append(line_ref)); } super::consts::TIME_STAMP => {} _ => { @@ -143,7 +157,7 @@ impl DbJsonEntity { super::consts::TIME_STAMP_LOWER_CASE, ) { } else { - raw.append(src, &line); + raw.append(line_ref); } } } @@ -159,7 +173,10 @@ impl DbJsonEntity { return Err(DbEntityParseFail::PartitionKeyIsTooLong); } - if partition_key.value.is_null(src) { + if partition_key + .value + .is_null(json_first_line_reader.get_src_slice()) + { return Err(DbEntityParseFail::FieldPartitionKeyCanNotBeNull); } @@ -169,7 +186,10 @@ impl DbJsonEntity { let row_key = row_key.unwrap(); - if row_key.value.is_null(src) { + if row_key + .value + .is_null(json_first_line_reader.get_src_slice()) + { return Err(DbEntityParseFail::FieldRowKeyCanNotBeNull); } @@ -209,9 +229,12 @@ impl DbJsonEntity { None } - pub fn restore_into_db_row(raw: &[u8]) -> Result { - let db_row = Self::new(raw)?; - let result = DbRow::new(db_row, raw.to_vec()); + pub fn restore_into_db_row( + json_first_line_reader: JsonFirstLineReader, + ) -> Result { + let raw = json_first_line_reader.get_src_slice().to_vec(); + let db_row = Self::new(json_first_line_reader)?; + let result = DbRow::new(db_row, raw); Ok(result) } @@ -221,9 +244,16 @@ impl DbJsonEntity { ) -> Result>, DbEntityParseFail> { let mut result = Vec::new(); - for json in src.split_array_json_to_objects() { + let slice_iterator = SliceIterator::new(src); + + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + + while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_row = DbJsonEntity::parse_into_db_row(json, inject_time_stamp)?; + let db_row = DbJsonEntity::parse_into_db_row( + json.unwrap_as_object(&json_array_iterator).unwrap(), + inject_time_stamp, + )?; result.push(Arc::new(db_row)); } return Ok(result); @@ -232,9 +262,15 @@ impl DbJsonEntity { pub fn restore_as_vec(src: &[u8]) -> Result>, DbEntityParseFail> { let mut result = Vec::new(); - for json in src.split_array_json_to_objects() { + let slice_iterator = SliceIterator::new(src); + + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + + while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_entity = DbJsonEntity::restore_into_db_row(json)?; + let db_entity = DbJsonEntity::restore_into_db_row( + json.unwrap_as_object(&json_array_iterator).unwrap(), + )?; result.push(Arc::new(db_entity)); } return Ok(result); @@ -246,9 +282,16 @@ impl DbJsonEntity { ) -> Result>)>, DbEntityParseFail> { let mut result = Vec::new(); - for json in src.split_array_json_to_objects() { + let slice_iterator = SliceIterator::new(src); + + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + + while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_row = DbJsonEntity::parse_into_db_row(json, inject_time_stamp)?; + let db_row = DbJsonEntity::parse_into_db_row( + json.unwrap_as_object(&json_array_iterator).unwrap(), + inject_time_stamp, + )?; let partition_key = db_row.get_partition_key(); @@ -272,9 +315,14 @@ impl DbJsonEntity { ) -> Result>)>, DbEntityParseFail> { let mut result = Vec::new(); - for json in src.split_array_json_to_objects() { + let slice_iterator = SliceIterator::new(src); + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + + while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_row = DbJsonEntity::restore_into_db_row(json)?; + let db_row = DbJsonEntity::restore_into_db_row( + json.unwrap_as_object(&json_array_iterator).unwrap(), + )?; let partition_key = db_row.get_partition_key(); @@ -386,7 +434,7 @@ fn inject_time_stamp_key_value( pub fn get_the_end_of_the_json(data: &[u8]) -> usize { for i in (0..data.len()).rev() { - if data[i] == my_json::json_reader::consts::CLOSE_BRACKET { + if data[i] == my_json::consts::CLOSE_BRACKET { return i; } } @@ -397,12 +445,47 @@ pub fn get_the_end_of_the_json(data: &[u8]) -> usize { #[cfg(test)] mod tests { - use rust_extensions::date_time::DateTimeAsMicroseconds; + use my_json::json_reader::JsonFirstLineReader; + use rust_extensions::{ + array_of_bytes_iterator::SliceIterator, date_time::DateTimeAsMicroseconds, + }; use crate::db_json_entity::{DbEntityParseFail, JsonTimeStamp}; use super::DbJsonEntity; + #[test] + pub fn test_partition_key_and_row_key_and_time_stamp_are_ok() { + let src_json = r#"{"TwoFaMethods": {}, + "PartitionKey": "ff95cdae9f7e4f1a847f6b83ad68b495", + "RowKey": "6c09c7f0e44d4ef79cfdd4252ebd54ab", + "TimeStamp": "2022-03-17T09:28:27.5923", + "Expires": "2022-03-17T13:28:29.6537478Z" + }"#; + + let json_first_line_reader: JsonFirstLineReader = src_json.into(); + + let json_time = JsonTimeStamp::now(); + + let entity = DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_time).unwrap(); + + let json_first_line_reader: JsonFirstLineReader = + entity.get_src_as_slice().into(); + + let dest_entity = + DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_time).unwrap(); + + assert_eq!( + "ff95cdae9f7e4f1a847f6b83ad68b495", + dest_entity.get_partition_key() + ); + + assert_eq!( + "6c09c7f0e44d4ef79cfdd4252ebd54ab", + dest_entity.get_row_key() + ); + } + #[test] pub fn parse_expires_with_z() { let src_json = r#"{"TwoFaMethods": {}, @@ -412,7 +495,9 @@ mod tests { "Expires": "2022-03-17T13:28:29.6537478Z" }"#; - let entity = DbJsonEntity::new(src_json.as_bytes()).unwrap(); + let json_first_line_reader: JsonFirstLineReader> = src_json.into(); + + let entity = DbJsonEntity::new(json_first_line_reader).unwrap(); let expires = entity.expires_value.as_ref().unwrap(); @@ -443,7 +528,10 @@ mod tests { "Expires": "2022-03-17T13:28:29.6537478Z" }"#; - let result = DbJsonEntity::new(src_json.as_bytes()); + let slice_iterator = SliceIterator::new(src_json.as_bytes()); + let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + + let result = DbJsonEntity::new(json_first_line_reader); if let Err(DbEntityParseFail::FieldPartitionKeyCanNotBeNull) = result { } else { @@ -455,7 +543,9 @@ mod tests { let src_json = r#"{"value":{"is_enabled":true,"fee_percent":5.0,"min_balance_usd":100.0,"fee_period_days":30,"inactivity_period_days":90},"PartitionKey":"*","RowKey":"*"}"#; let time_stamp = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(src_json.as_bytes(), &time_stamp).unwrap(); + let slice_iterator = SliceIterator::new(src_json.as_bytes()); + let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let db_row = DbJsonEntity::parse_into_db_row(json_first_line_reader, &time_stamp).unwrap(); println!( "{:?}", @@ -471,7 +561,10 @@ mod tests { let mut json = r#"{"PartitionKey":"PK", "RowKey":"RK"} "#.as_bytes().to_vec(); - let mut db_json_entity = DbJsonEntity::new(json.as_slice()).unwrap(); + let slice_iterator = SliceIterator::new(json.as_slice()); + let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + + let mut db_json_entity = DbJsonEntity::new(json_first_line_reader).unwrap(); db_json_entity.inject_at_the_end_of_json(&mut json, &json_ts); @@ -500,7 +593,10 @@ mod tests { let json = r#"{"PartitionKey":"Pk", "RowKey":"Rk", "timestamp":null}"#; - let db_row = DbJsonEntity::parse_into_db_row(json.as_bytes(), &json_ts).unwrap(); + let slice_iterator = SliceIterator::new(json.as_bytes()); + let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + + let db_row = DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_ts).unwrap(); assert_eq!(db_row.get_partition_key(), "Pk",); assert_eq!(db_row.get_row_key(), "Rk",); @@ -514,7 +610,11 @@ mod tests { let json = r#"{"PartitionKey":"Pk", "RowKey":"Rk", "timestamp":"12345678901234567890123456789012345678901234567890"}"#; - let db_json_entity = DbJsonEntity::parse_into_db_row(json.as_bytes(), &json_ts).unwrap(); + let slice_iterator = SliceIterator::new(json.as_bytes()); + let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + + let db_json_entity = + DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_ts).unwrap(); assert_eq!(db_json_entity.get_partition_key(), "Pk",); assert_eq!(db_json_entity.get_row_key(), "Rk",); @@ -532,8 +632,11 @@ mod tests { let inject_time_stamp = JsonTimeStamp::now(); + let slice_iterator = SliceIterator::new(test_json.as_bytes()); + let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(json_first_line_reader, &inject_time_stamp).unwrap(); assert_eq!(db_row.get_partition_key(), "Pk"); assert_eq!(db_row.get_row_key(), "Rk"); diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs b/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs index b75c4a3..b83a7a0 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs @@ -1,3 +1,6 @@ +use my_json::json_reader::JsonFirstLineReader; +use rust_extensions::array_of_bytes_iterator::SliceIterator; + use crate::db::DbRow; use super::{DbEntityParseFail, DbJsonEntity, JsonTimeStamp}; @@ -34,6 +37,8 @@ impl<'s> DbJsonEntityWithContent<'s> { } pub fn into_db_row(self) -> Result { - DbJsonEntity::parse_into_db_row(self.raw, &self.time_stamp) + let slice_iterator = SliceIterator::new(self.raw); + let first_line_reader = JsonFirstLineReader::new(slice_iterator); + DbJsonEntity::parse_into_db_row(first_line_reader, &self.time_stamp) } } diff --git a/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs b/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs index e1ae670..4b8817e 100644 --- a/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs +++ b/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs @@ -1,4 +1,4 @@ -use my_json::json_reader::JsonFirstLine; +use my_json::json_reader::JsonKeyValueRef; use crate::db_json_entity::{JsonKeyValuePosition, KeyValueContentPosition}; @@ -37,7 +37,7 @@ impl DbRowContentCompiler { } } - pub fn append(&mut self, src: &[u8], line: &JsonFirstLine) -> JsonKeyValuePosition { + pub fn append(&mut self, line: JsonKeyValueRef) -> JsonKeyValuePosition { self.append_first_line(); let mut key = KeyValueContentPosition { start: self.content.len(), @@ -45,7 +45,7 @@ impl DbRowContentCompiler { }; self.content - .extend_from_slice(&src[line.name_start..line.name_end]); + .extend_from_slice(line.name.as_raw_str().unwrap().as_bytes()); key.end = self.content.len(); @@ -55,8 +55,7 @@ impl DbRowContentCompiler { start: self.content.len(), end: 0, }; - self.content - .extend_from_slice(&src[line.value_start..line.value_end]); + self.content.extend_from_slice(line.value.as_bytes()); value.end = self.content.len(); diff --git a/my-no-sql-core/src/db_json_entity/json_key_value_position.rs b/my-no-sql-core/src/db_json_entity/json_key_value_position.rs index cc07434..91574ee 100644 --- a/my-no-sql-core/src/db_json_entity/json_key_value_position.rs +++ b/my-no-sql-core/src/db_json_entity/json_key_value_position.rs @@ -1,4 +1,4 @@ -use my_json::json_reader::JsonFirstLine; +use my_json::json_reader::JsonKeyValue; #[derive(Debug, Clone)] pub struct KeyValueContentPosition { @@ -31,16 +31,16 @@ pub struct JsonKeyValuePosition { } impl JsonKeyValuePosition { - pub fn new(src: &JsonFirstLine) -> Self { + pub fn new(src: &JsonKeyValue) -> Self { Self { key: KeyValueContentPosition { - start: src.name_start, - end: src.name_end, + start: src.name.start, + end: src.name.end, }, value: KeyValueContentPosition { - start: src.value_start, - end: src.value_end, + start: src.value.start, + end: src.value.end, }, } } diff --git a/my-no-sql-core/src/entity_serializer.rs b/my-no-sql-core/src/entity_serializer.rs index 5362c60..86dd1e1 100644 --- a/my-no-sql-core/src/entity_serializer.rs +++ b/my-no-sql-core/src/entity_serializer.rs @@ -1,4 +1,6 @@ +use my_json::json_reader::JsonFirstLineReader; use my_no_sql_abstractions::MyNoSqlEntity; +use rust_extensions::array_of_bytes_iterator::SliceIterator; use serde::{de::DeserializeOwned, Serialize}; use crate::db_json_entity::DbJsonEntity; @@ -19,7 +21,9 @@ where match parse_result { Ok(el) => return el, Err(err) => { - let db_entity = DbJsonEntity::new(data); + let slice_iterator = SliceIterator::new(data); + let json_first_line_iterator = JsonFirstLineReader::new(slice_iterator); + let db_entity = DbJsonEntity::new(json_first_line_iterator); match db_entity { Ok(db_entity) => { @@ -57,14 +61,14 @@ pub fn inject_partition_key_and_row_key( let to_insert = if let Some(row_key) = row_key { format!( "\"PartitionKey\":\"{}\",\"RowKey\":\"{}\",", - my_json::EscapedJsonString::new(partition_key).as_str(), - my_json::EscapedJsonString::new(row_key).as_str() + my_json::json_string_value::escape_json_string_value(partition_key).as_str(), + my_json::json_string_value::escape_json_string_value(row_key).as_str(), ) .into_bytes() } else { format!( "\"PartitionKey\":\"{}\",", - my_json::EscapedJsonString::new(partition_key).as_str(), + my_json::json_string_value::escape_json_string_value(partition_key).as_str(), ) .into_bytes() }; diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index b4f880b..42bceb3 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -15,7 +15,7 @@ rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-ext ] } flurl = { tag = "0.5.2", git = "https://github.com/MyJetTools/fl-url" } -my-json = { tag = "0.2.2", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.2.4", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index bdfbab4..3b69c03 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -2,11 +2,12 @@ use std::sync::Arc; use flurl::{FlUrl, FlUrlResponse}; use my_json::{ - json_reader::array_parser::JsonArrayIterator, + json_reader::array_iterator::JsonArrayIterator, json_writer::{JsonArrayWriter, RawJsonObject}, }; use my_logger::LogEventCtx; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; +use rust_extensions::array_of_bytes_iterator::SliceIterator; use crate::{ CreateTableParams, DataWriterError, MyNoSqlWriterSettings, OperationFailHttpContract, @@ -527,12 +528,28 @@ fn deserialize_entities( src: &[u8], ) -> Result, DataWriterError> { let mut result = Vec::new(); + let slice_iterator = SliceIterator::new(src); + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + + while let Some(item) = json_array_iterator.get_next() { + let itm = item.unwrap(); + + result.push(TEntity::deserialize_entity(itm.as_bytes(&json_array_iterator)).unwrap()); + } + Ok(result) + + /* + let mut result = Vec::new(); + + + for itm in JsonArrayIterator::new(src) { let itm = itm.unwrap(); result.push(TEntity::deserialize_entity(itm).unwrap()); } Ok(result) + */ } async fn create_table_errors_handler( @@ -591,7 +608,7 @@ mod tests { fn deserialize_entity(src: &[u8]) -> Option { let result: Self = my_no_sql_core::entity_serializer::deserialize(src); - Some(result) + result.into() } } diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index 2da2e6b..bda750b 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -111,7 +111,7 @@ pub fn get_fn_standard_serialize_deserialize() -> proc_macro2::TokenStream { fn deserialize_entity(src: &[u8]) -> Option { - let result: Self = my_no_sql_sdk::core::entity_serializer::deserialize(src); + let result = my_no_sql_sdk::core::entity_serializer::deserialize(src); Some(result) } } diff --git a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs index 098477e..5e9edb3 100644 --- a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs @@ -161,7 +161,7 @@ fn get_deserialize_cases(enum_cases: &[EnumCase]) -> Result Option { - let result: Self = my_no_sql_core::entity_serializer::deserialize(src); + let result = my_no_sql_core::entity_serializer::deserialize(src); Some(result) } } diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index a913d3f..733c995 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -1,10 +1,10 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration}; use async_trait::async_trait; -use my_json::json_reader::array_parser::JsonArrayIterator; +use my_json::json_reader::array_iterator::JsonArrayIterator; use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::SyncToMainNodeHandler; -use rust_extensions::{ApplicationStates, StrOrString}; +use rust_extensions::{array_of_bytes_iterator::SliceIterator, ApplicationStates, StrOrString}; use serde::de::DeserializeOwned; use tokio::sync::RwLock; @@ -109,6 +109,40 @@ where } pub fn deserialize_array(&self, data: &[u8]) -> BTreeMap> { + let slice_iterator = SliceIterator::new(data); + + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + + let mut result = BTreeMap::new(); + + while let Some(db_entity) = json_array_iterator.get_next() { + if let Err(err) = &db_entity { + panic!( + "Table: {}. The whole array of json entities is broken. Err: {:?}", + TMyNoSqlEntity::TABLE_NAME, + err + ); + } + + let db_entity_data = db_entity.unwrap(); + + let el = + TMyNoSqlEntity::deserialize_entity(db_entity_data.as_bytes(&json_array_iterator)); + + if let Some(el) = el { + let partition_key = el.get_partition_key(); + if !result.contains_key(partition_key) { + result.insert(partition_key.to_string(), Vec::new()); + } + + result.get_mut(partition_key).unwrap().push(el); + } + } + + result + + /* + let mut result = BTreeMap::new(); for db_entity in JsonArrayIterator::new(data) { @@ -135,6 +169,7 @@ where } result + */ } pub async fn get_enum_case_models_by_partition_key< From bd9e3e5a2a232921beb6796f703498697da9f3ac Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 1 Aug 2024 19:49:57 +0300 Subject: [PATCH 06/89] Added ability to do request through SSH --- my-no-sql-abstractions/Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 2 +- my-no-sql-data-writer/Cargo.toml | 8 +- .../src/my_no_sql_data_writer/execution.rs | 84 ++++++---------- .../src/my_no_sql_data_writer/fl_url_ext.rs | 3 - .../my_no_sql_data_writer/fl_url_factory.rs | 84 ++++++++++++++++ .../src/my_no_sql_data_writer/mod.rs | 1 + .../my_no_sql_data_writer.rs | 91 +++++++++++------- .../src/my_no_sql_data_writer/with_retries.rs | 95 ++++++++++--------- my-no-sql-macros/Cargo.toml | 2 +- my-no-sql-sdk/Cargo.toml | 2 +- my-no-sql-tcp-reader/Cargo.toml | 2 +- my-no-sql-tcp-shared/Cargo.toml | 2 +- my-no-sql-tests/Cargo.toml | 2 +- 14 files changed, 233 insertions(+), 147 deletions(-) create mode 100644 my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs diff --git a/my-no-sql-abstractions/Cargo.toml b/my-no-sql-abstractions/Cargo.toml index 28cd9bb..472d191 100644 --- a/my-no-sql-abstractions/Cargo.toml +++ b/my-no-sql-abstractions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-abstractions" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index 2082449..b773f23 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-core" -version = "0.3.1" +version = "0.3.2" edition = "2021" [features] diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 42bceb3..162b9b9 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -1,11 +1,15 @@ [package] name = "my-no-sql-data-writer" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = [] +with-ssh = ["flurl/with-ssh"] + [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" } @@ -13,7 +17,7 @@ my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ "with-tokio", ] } -flurl = { tag = "0.5.2", git = "https://github.com/MyJetTools/fl-url" } +flurl = { tag = "0.5.3", git = "https://github.com/MyJetTools/fl-url" } my-json = { tag = "0.2.4", git = "https://github.com/MyJetTools/my-json.git" } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 3b69c03..ab86c6b 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use flurl::{FlUrl, FlUrlResponse}; use my_json::{ json_reader::array_iterator::JsonArrayIterator, @@ -9,10 +7,7 @@ use my_logger::LogEventCtx; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::array_of_bytes_iterator::SliceIterator; -use crate::{ - CreateTableParams, DataWriterError, MyNoSqlWriterSettings, OperationFailHttpContract, - UpdateReadStatistics, -}; +use crate::{CreateTableParams, DataWriterError, OperationFailHttpContract, UpdateReadStatistics}; use super::fl_url_ext::FlUrlExt; @@ -21,13 +16,13 @@ const ROWS_CONTROLLER: &str = "Rows"; const BULK_CONTROLLER: &str = "Bulk"; pub async fn create_table_if_not_exists( - settings: Arc, + flurl: FlUrl, + url: &str, table_name: &'static str, - params: CreateTableParams, + params: &CreateTableParams, sync_period: DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { - let url = settings.get_url().await; - let fl_url = FlUrl::new(url.clone()) + let fl_url = flurl .append_path_segment("Tables") .append_path_segment("CreateIfNotExists") .append_data_sync_period(&sync_period) @@ -37,18 +32,16 @@ pub async fn create_table_if_not_exists( let mut response = fl_url.post(None).await?; - create_table_errors_handler(&mut response, "create_table_if_not_exists", url.as_str()).await + create_table_errors_handler(&mut response, "create_table_if_not_exists", url).await } pub async fn create_table( - settings: &Arc, + flurl: FlUrl, + url: &str, table_name: &str, params: CreateTableParams, sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { - let url = settings.get_url().await; - let flurl = FlUrl::new(url.as_str()); - let fl_url = flurl .append_path_segment("Tables") .append_path_segment("Create") @@ -59,16 +52,14 @@ pub async fn create_table( let mut response = fl_url.post(None).await?; - create_table_errors_handler(&mut response, "create_table", url.as_str()).await + create_table_errors_handler(&mut response, "create_table", url).await } pub async fn insert_entity( - settings: &Arc, + flurl: FlUrl, entity: &TEntity, sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { - let flurl = get_fl_url(settings).await; - let response = flurl .append_path_segment(ROW_CONTROLLER) .append_path_segment("Insert") @@ -89,12 +80,10 @@ pub async fn insert_entity( - settings: &Arc, + flurl: FlUrl, entity: &TEntity, sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { - let flurl = get_fl_url(settings).await; - let response = flurl .append_path_segment(ROW_CONTROLLER) .append_path_segment("InsertOrReplace") @@ -115,7 +104,7 @@ pub async fn insert_or_replace_entity< pub async fn bulk_insert_or_replace< TEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send, >( - settings: &Arc, + flurl: FlUrl, entities: &[TEntity], sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { @@ -123,8 +112,6 @@ pub async fn bulk_insert_or_replace< return Ok(()); } - let flurl = get_fl_url(settings).await; - let response = flurl .append_path_segment(BULK_CONTROLLER) .append_path_segment("InsertOrReplace") @@ -143,13 +130,11 @@ pub async fn bulk_insert_or_replace< } pub async fn get_entity( - settings: &Arc, + flurl: FlUrl, partition_key: &str, row_key: &str, update_read_statistics: Option<&UpdateReadStatistics>, ) -> Result, DataWriterError> { - let flurl = get_fl_url(settings).await; - let mut request = flurl .append_path_segment(ROW_CONTROLLER) .with_partition_key_as_query_param(partition_key) @@ -179,11 +164,10 @@ pub async fn get_entity( - settings: &Arc, + flurl: FlUrl, partition_key: &str, update_read_statistics: Option<&UpdateReadStatistics>, ) -> Result>, DataWriterError> { - let flurl = get_fl_url(settings).await; let mut request = flurl .append_path_segment(ROW_CONTROLLER) .with_partition_key_as_query_param(partition_key) @@ -218,11 +202,11 @@ pub async fn get_enum_case_models_by_partition_key< + Send + 'static, >( - settings: &Arc, + flurl: FlUrl, update_read_statistics: Option<&UpdateReadStatistics>, ) -> Result>, DataWriterError> { let result: Option> = - get_by_partition_key(settings, TResult::PARTITION_KEY, update_read_statistics).await?; + get_by_partition_key(flurl, TResult::PARTITION_KEY, update_read_statistics).await?; match result { Some(entities) => { @@ -247,11 +231,11 @@ pub async fn get_enum_case_model< + Send + 'static, >( - settings: &Arc, + flurl: FlUrl, update_read_statistics: Option<&UpdateReadStatistics>, ) -> Result, DataWriterError> { let entity: Option = get_entity( - settings, + flurl, TResult::PARTITION_KEY, TResult::ROW_KEY, update_read_statistics, @@ -265,11 +249,9 @@ pub async fn get_enum_case_model< } pub async fn get_by_row_key( - settings: &Arc, + flurl: FlUrl, row_key: &str, ) -> Result>, DataWriterError> { - let flurl = get_fl_url(settings).await; - let mut response = flurl .append_path_segment(ROW_CONTROLLER) .with_row_key_as_query_param(row_key) @@ -300,10 +282,10 @@ pub async fn delete_enum_case< + Send + 'static, >( - settings: &Arc, + flurl: FlUrl, ) -> Result, DataWriterError> { let entity: Option = - delete_row(settings, TResult::PARTITION_KEY, TResult::ROW_KEY).await?; + delete_row(flurl, TResult::PARTITION_KEY, TResult::ROW_KEY).await?; match entity { Some(entity) => Ok(Some(entity.into())), @@ -320,10 +302,10 @@ pub async fn delete_enum_case_with_row_key< + Send + 'static, >( - settings: &Arc, + flurl: FlUrl, row_key: &str, ) -> Result, DataWriterError> { - let entity: Option = delete_row(settings, TResult::PARTITION_KEY, row_key).await?; + let entity: Option = delete_row(flurl, TResult::PARTITION_KEY, row_key).await?; match entity { Some(entity) => Ok(Some(entity.into())), @@ -332,11 +314,10 @@ pub async fn delete_enum_case_with_row_key< } pub async fn delete_row( - settings: &Arc, + flurl: FlUrl, partition_key: &str, row_key: &str, ) -> Result, DataWriterError> { - let flurl = get_fl_url(settings).await; let mut response = flurl .append_path_segment(ROW_CONTROLLER) .with_partition_key_as_query_param(partition_key) @@ -360,11 +341,10 @@ pub async fn delete_row, + flurl: FlUrl, table_name: &str, partition_keys: &[&str], ) -> Result<(), DataWriterError> { - let flurl = get_fl_url(settings).await; let mut response = flurl .append_path_segment(ROWS_CONTROLLER) .with_table_name_as_query_param(table_name) @@ -382,9 +362,8 @@ pub async fn delete_partitions( } pub async fn get_all( - settings: &Arc, + flurl: FlUrl, ) -> Result>, DataWriterError> { - let flurl = get_fl_url(settings).await; let mut response = flurl .append_path_segment(ROW_CONTROLLER) .with_table_name_as_query_param(TEntity::TABLE_NAME) @@ -408,11 +387,10 @@ pub async fn get_all( - settings: &Arc, + flurl: FlUrl, entities: &[TEntity], sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { - let flurl = get_fl_url(settings).await; let mut response = flurl .append_path_segment(BULK_CONTROLLER) .append_path_segment("CleanAndBulkInsert") @@ -429,12 +407,11 @@ pub async fn clean_table_and_bulk_insert< pub async fn clean_partition_and_bulk_insert< TEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send, >( - settings: &Arc, + flurl: FlUrl, partition_key: &str, entities: &[TEntity], sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { - let flurl = get_fl_url(settings).await; let mut response = flurl .append_path_segment(BULK_CONTROLLER) .append_path_segment("CleanAndBulkInsert") @@ -453,11 +430,6 @@ fn is_ok_result(response: &FlUrlResponse) -> bool { response.get_status_code() >= 200 && response.get_status_code() < 300 } -async fn get_fl_url(settings: &Arc) -> FlUrl { - let url = settings.get_url().await; - FlUrl::new(url) -} - fn serialize_entities_to_body( entities: &[TEntity], ) -> Option> { diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs index fff9f34..a06a041 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs @@ -3,13 +3,10 @@ use my_no_sql_abstractions::DataSynchronizationPeriod; pub trait FlUrlExt { fn with_table_name_as_query_param(self, table_name: &str) -> FlUrl; - fn append_data_sync_period(self, sync_period: &DataSynchronizationPeriod) -> FlUrl; - fn with_partition_key_as_query_param(self, partition_key: &str) -> FlUrl; fn with_partition_keys_as_query_param(self, partition_keys: &[&str]) -> FlUrl; fn with_row_key_as_query_param(self, partition_key: &str) -> FlUrl; - fn with_persist_as_query_param(self, persist: bool) -> FlUrl; } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs new file mode 100644 index 0000000..e1049fd --- /dev/null +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -0,0 +1,84 @@ +use std::sync::Arc; + +use flurl::{ + my_ssh::{SshCredentials, SshSessionsPool}, + FlUrl, +}; +use rust_extensions::UnsafeValue; + +use super::{CreateTableParams, DataWriterError, MyNoSqlWriterSettings}; + +#[derive(Clone)] +pub struct FlUrlFactory { + settings: Arc, + auto_create_table_params: Option>, + #[cfg(feature = "with-ssh")] + pub ssh_credentials: Option>, + #[cfg(feature = "with-ssh")] + pub ssh_sessions_pool: Option>, + create_table_is_called: Arc>, + table_name: &'static str, +} + +impl FlUrlFactory { + pub fn new( + settings: Arc, + auto_create_table_params: Option>, + table_name: &'static str, + ) -> Self { + Self { + auto_create_table_params, + ssh_credentials: None, + ssh_sessions_pool: None, + create_table_is_called: UnsafeValue::new(false).into(), + settings, + table_name, + } + } + + async fn create_fl_url(&self, url: &str) -> FlUrl { + let mut fl_url = flurl::FlUrl::new(url); + + if let Some(ssh_credentials) = &self.ssh_credentials { + fl_url = fl_url.set_ssh_credentials(ssh_credentials.clone()); + } + + if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { + fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); + } + + fl_url + } + + pub async fn get_fl_url(&self) -> Result<(FlUrl, String), DataWriterError> { + let url = self.settings.get_url().await; + if !self.create_table_is_called.get_value() { + if let Some(crate_table_params) = &self.auto_create_table_params { + self.create_table_if_not_exists(url.as_str(), crate_table_params) + .await?; + } + + self.create_table_is_called.set_value(true); + } + + let result = self.create_fl_url(url.as_str()).await; + + Ok((result, url)) + } + + pub async fn create_table_if_not_exists( + &self, + url: &str, + create_table_params: &CreateTableParams, + ) -> Result<(), DataWriterError> { + let fl_url = self.create_fl_url(url).await; + super::execution::create_table_if_not_exists( + fl_url, + url, + self.table_name, + create_table_params, + my_no_sql_abstractions::DataSynchronizationPeriod::Sec1, + ) + .await + } +} diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs index fbff7eb..76d6049 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs @@ -10,3 +10,4 @@ mod execution; mod fl_url_ext; mod with_retries; pub use with_retries::*; +mod fl_url_factory; diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 564c4ed..666abdc 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -1,13 +1,17 @@ use std::{marker::PhantomData, sync::Arc, time::Duration}; use flurl::FlUrl; + use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; +#[cfg(feature = "with-ssh")] +use flurl::my_ssh::*; + use serde::{Deserialize, Serialize}; use crate::{MyNoSqlDataWriterWithRetries, MyNoSqlWriterSettings}; -use super::{DataWriterError, UpdateReadStatistics}; +use super::{fl_url_factory::FlUrlFactory, DataWriterError, UpdateReadStatistics}; pub struct CreateTableParams { pub persist: bool, @@ -40,9 +44,9 @@ impl CreateTableParams { } pub struct MyNoSqlDataWriter { - settings: Arc, sync_period: DataSynchronizationPeriod, phantom: PhantomData, + fl_url_factory: FlUrlFactory, } impl MyNoSqlDataWriter { @@ -51,25 +55,23 @@ impl MyNoSqlData auto_create_table_params: Option, sync_period: DataSynchronizationPeriod, ) -> Self { - if let Some(create_table_params) = auto_create_table_params { - tokio::spawn(super::execution::create_table_if_not_exists( - settings.clone(), - TEntity::TABLE_NAME, - create_table_params, - sync_period, - )); - } - Self { - settings, phantom: PhantomData, sync_period, + fl_url_factory: FlUrlFactory::new( + settings, + auto_create_table_params.map(|itm| itm.into()), + TEntity::TABLE_NAME, + ), } } pub async fn create_table(&self, params: CreateTableParams) -> Result<(), DataWriterError> { + let (fl_url, url) = self.fl_url_factory.get_fl_url().await?; + super::execution::create_table( - &self.settings, + fl_url, + url.as_str(), TEntity::TABLE_NAME, params, &self.sync_period, @@ -77,12 +79,22 @@ impl MyNoSqlData .await } + pub async fn set_ssh_credentials(&mut self, ssh_credentials: Arc) { + self.fl_url_factory.ssh_credentials = Some(ssh_credentials); + } + + pub async fn set_ssh_sessions_pool(&mut self, ssh_sessions_pool: Arc) { + self.fl_url_factory.ssh_sessions_pool = Some(ssh_sessions_pool); + } + pub async fn create_table_if_not_exists( &self, - params: CreateTableParams, + params: &CreateTableParams, ) -> Result<(), DataWriterError> { + let (fl_url, url) = self.fl_url_factory.get_fl_url().await?; super::execution::create_table_if_not_exists( - self.settings.clone(), + fl_url, + url.as_str(), TEntity::TABLE_NAME, params, self.sync_period, @@ -96,7 +108,7 @@ impl MyNoSqlData max_attempts: usize, ) -> MyNoSqlDataWriterWithRetries { MyNoSqlDataWriterWithRetries::new( - self.settings.clone(), + self.fl_url_factory.clone(), self.sync_period, delay_between_attempts, max_attempts, @@ -104,18 +116,21 @@ impl MyNoSqlData } pub async fn insert_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { - super::execution::insert_entity(&self.settings, entity, &self.sync_period).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::insert_entity(fl_url, entity, &self.sync_period).await } pub async fn insert_or_replace_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { - super::execution::insert_or_replace_entity(&self.settings, entity, &self.sync_period).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::insert_or_replace_entity(fl_url, entity, &self.sync_period).await } pub async fn bulk_insert_or_replace( &self, entities: &[TEntity], ) -> Result<(), DataWriterError> { - super::execution::bulk_insert_or_replace(&self.settings, entities, &self.sync_period).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::bulk_insert_or_replace(fl_url, entities, &self.sync_period).await } pub async fn get_entity( @@ -124,8 +139,9 @@ impl MyNoSqlData row_key: &str, update_read_statistics: Option, ) -> Result, DataWriterError> { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; super::execution::get_entity( - &self.settings, + fl_url, partition_key, row_key, update_read_statistics.as_ref(), @@ -138,8 +154,9 @@ impl MyNoSqlData partition_key: &str, update_read_statistics: Option, ) -> Result>, DataWriterError> { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; super::execution::get_by_partition_key( - &self.settings, + fl_url, partition_key, update_read_statistics.as_ref(), ) @@ -157,8 +174,9 @@ impl MyNoSqlData &self, update_read_statistics: Option, ) -> Result>, DataWriterError> { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; super::execution::get_enum_case_models_by_partition_key( - &self.settings, + fl_url, update_read_statistics.as_ref(), ) .await @@ -175,14 +193,16 @@ impl MyNoSqlData &self, update_read_statistics: Option, ) -> Result, DataWriterError> { - super::execution::get_enum_case_model(&self.settings, update_read_statistics.as_ref()).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::get_enum_case_model(fl_url, update_read_statistics.as_ref()).await } pub async fn get_by_row_key( &self, row_key: &str, ) -> Result>, DataWriterError> { - super::execution::get_by_row_key(&self.settings, row_key).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::get_by_row_key(fl_url, row_key).await } pub async fn delete_enum_case< @@ -195,7 +215,8 @@ impl MyNoSqlData >( &self, ) -> Result, DataWriterError> { - super::execution::delete_enum_case(&self.settings).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::delete_enum_case(fl_url).await } pub async fn delete_enum_case_with_row_key< @@ -209,7 +230,8 @@ impl MyNoSqlData &self, row_key: &str, ) -> Result, DataWriterError> { - super::execution::delete_enum_case_with_row_key(&self.settings, row_key).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::delete_enum_case_with_row_key(fl_url, row_key).await } pub async fn delete_row( @@ -217,24 +239,26 @@ impl MyNoSqlData partition_key: &str, row_key: &str, ) -> Result, DataWriterError> { - super::execution::delete_row(&self.settings, partition_key, row_key).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::delete_row(fl_url, partition_key, row_key).await } pub async fn delete_partitions(&self, partition_keys: &[&str]) -> Result<(), DataWriterError> { - super::execution::delete_partitions(&self.settings, TEntity::TABLE_NAME, partition_keys) - .await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::delete_partitions(fl_url, TEntity::TABLE_NAME, partition_keys).await } pub async fn get_all(&self) -> Result>, DataWriterError> { - super::execution::get_all(&self.settings).await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::get_all(fl_url).await } pub async fn clean_table_and_bulk_insert( &self, entities: &[TEntity], ) -> Result<(), DataWriterError> { - super::execution::clean_table_and_bulk_insert(&self.settings, entities, &self.sync_period) - .await + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::clean_table_and_bulk_insert(fl_url, entities, &self.sync_period).await } pub async fn clean_partition_and_bulk_insert( @@ -242,8 +266,9 @@ impl MyNoSqlData partition_key: &str, entities: &[TEntity], ) -> Result<(), DataWriterError> { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; super::execution::clean_partition_and_bulk_insert( - &self.settings, + fl_url, partition_key, entities, &self.sync_period, diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs index 3e83836..b85dcfe 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs @@ -1,11 +1,13 @@ -use std::{marker::PhantomData, sync::Arc, time::Duration}; +use std::{marker::PhantomData, time::Duration}; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; -use crate::{DataWriterError, MyNoSqlWriterSettings, UpdateReadStatistics}; +use crate::{DataWriterError, UpdateReadStatistics}; + +use super::fl_url_factory::FlUrlFactory; pub struct MyNoSqlDataWriterWithRetries { - settings: Arc, + fl_url_factory: FlUrlFactory, sync_period: DataSynchronizationPeriod, attempt_delay: Duration, phantom: PhantomData, @@ -16,25 +18,26 @@ impl MyNoSqlDataWriterWithRetries { pub fn new( - settings: Arc, + fl_url_factory: FlUrlFactory, sync_period: DataSynchronizationPeriod, attempt_delay: Duration, max_attempts: usize, ) -> Self { Self { - settings, phantom: PhantomData, sync_period, attempt_delay, max_attempts, + fl_url_factory, } } pub async fn insert_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { let mut attempt_no = 0; loop { - let result = - super::execution::insert_entity(&self.settings, entity, &self.sync_period).await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + + let result = super::execution::insert_entity(fl_url, entity, &self.sync_period).await; if result.is_ok() { return result; @@ -54,12 +57,9 @@ impl pub async fn insert_or_replace_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::insert_or_replace_entity( - &self.settings, - entity, - &self.sync_period, - ) - .await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = + super::execution::insert_or_replace_entity(fl_url, entity, &self.sync_period).await; match result { Ok(result) => return Ok(result), @@ -78,12 +78,9 @@ impl ) -> Result<(), DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::bulk_insert_or_replace( - &self.settings, - entities, - &self.sync_period, - ) - .await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = + super::execution::bulk_insert_or_replace(fl_url, entities, &self.sync_period).await; match result { Ok(result) => return Ok(result), @@ -104,8 +101,10 @@ impl ) -> Result, DataWriterError> { let mut attempt_no = 0; loop { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = super::execution::get_entity( - &self.settings, + fl_url, partition_key, row_key, update_read_statistics.as_ref(), @@ -130,8 +129,9 @@ impl ) -> Result>, DataWriterError> { let mut attempt_no = 0; loop { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; let result = super::execution::get_by_partition_key( - &self.settings, + fl_url, partition_key, update_read_statistics.as_ref(), ) @@ -161,8 +161,9 @@ impl ) -> Result>, DataWriterError> { let mut attempt_no = 0; loop { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; let result = super::execution::get_enum_case_models_by_partition_key( - &self.settings, + fl_url, update_read_statistics.as_ref(), ) .await; @@ -191,11 +192,10 @@ impl ) -> Result, DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::get_enum_case_model( - &self.settings, - update_read_statistics.as_ref(), - ) - .await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = + super::execution::get_enum_case_model(fl_url, update_read_statistics.as_ref()) + .await; match result { Ok(result) => return Ok(result), @@ -214,7 +214,8 @@ impl ) -> Result>, DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::get_by_row_key(&self.settings, row_key).await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = super::execution::get_by_row_key(fl_url, row_key).await; match result { Ok(result) => return Ok(result), @@ -239,7 +240,8 @@ impl ) -> Result, DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::delete_enum_case(&self.settings).await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = super::execution::delete_enum_case(fl_url).await; match result { Ok(result) => return Ok(result), @@ -265,8 +267,8 @@ impl ) -> Result, DataWriterError> { let mut attempt_no = 0; loop { - let result = - super::execution::delete_enum_case_with_row_key(&self.settings, row_key).await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = super::execution::delete_enum_case_with_row_key(fl_url, row_key).await; match result { Ok(result) => return Ok(result), @@ -286,7 +288,8 @@ impl ) -> Result, DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::delete_row(&self.settings, partition_key, row_key).await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = super::execution::delete_row(fl_url, partition_key, row_key).await; match result { Ok(result) => return Ok(result), @@ -302,12 +305,11 @@ impl pub async fn delete_partitions(&self, partition_keys: &[&str]) -> Result<(), DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::delete_partitions( - &self.settings, - TEntity::TABLE_NAME, - partition_keys, - ) - .await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + + let result = + super::execution::delete_partitions(fl_url, TEntity::TABLE_NAME, partition_keys) + .await; match result { Ok(result) => return Ok(result), @@ -323,7 +325,9 @@ impl pub async fn get_all(&self) -> Result>, DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::get_all(&self.settings).await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + + let result = super::execution::get_all(fl_url).await; match result { Ok(result) => return Ok(result), @@ -342,12 +346,10 @@ impl ) -> Result<(), DataWriterError> { let mut attempt_no = 0; loop { - let result = super::execution::clean_table_and_bulk_insert( - &self.settings, - entities, - &self.sync_period, - ) - .await; + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let result = + super::execution::clean_table_and_bulk_insert(fl_url, entities, &self.sync_period) + .await; match result { Ok(result) => return Ok(result), @@ -367,8 +369,9 @@ impl ) -> Result<(), DataWriterError> { let mut attempt_no = 0; loop { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; let result = super::execution::clean_partition_and_bulk_insert( - &self.settings, + fl_url, partition_key, entities, &self.sync_period, diff --git a/my-no-sql-macros/Cargo.toml b/my-no-sql-macros/Cargo.toml index 5105e18..f5c2d4e 100644 --- a/my-no-sql-macros/Cargo.toml +++ b/my-no-sql-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-macros" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index b8c078e..d04745a 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-sdk" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index b4d9286..889c407 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-reader" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 4de2b0f..76db413 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-shared" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tests/Cargo.toml b/my-no-sql-tests/Cargo.toml index 0206965..ba0221e 100644 --- a/my-no-sql-tests/Cargo.toml +++ b/my-no-sql-tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tests" -version = "0.3.1" +version = "0.3.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 2d66cf1a2b2f69e1b3875f10a0d956e34c3b987c Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 1 Aug 2024 19:51:58 +0300 Subject: [PATCH 07/89] Made it possible to be buildable --- .../src/my_no_sql_data_writer/fl_url_factory.rs | 13 +++++++++---- .../my_no_sql_data_writer/my_no_sql_data_writer.rs | 2 ++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index e1049fd..af6e22c 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use flurl::{ - my_ssh::{SshCredentials, SshSessionsPool}, - FlUrl, -}; +use flurl::FlUrl; + +#[cfg(feature = "with-ssh")] +use flurl::my_ssh::{SshCredentials, SshSessionsPool}; + use rust_extensions::UnsafeValue; use super::{CreateTableParams, DataWriterError, MyNoSqlWriterSettings}; @@ -28,7 +29,9 @@ impl FlUrlFactory { ) -> Self { Self { auto_create_table_params, + #[cfg(feature = "with-ssh")] ssh_credentials: None, + #[cfg(feature = "with-ssh")] ssh_sessions_pool: None, create_table_is_called: UnsafeValue::new(false).into(), settings, @@ -39,10 +42,12 @@ impl FlUrlFactory { async fn create_fl_url(&self, url: &str) -> FlUrl { let mut fl_url = flurl::FlUrl::new(url); + #[cfg(feature = "with-ssh")] if let Some(ssh_credentials) = &self.ssh_credentials { fl_url = fl_url.set_ssh_credentials(ssh_credentials.clone()); } + #[cfg(feature = "with-ssh")] if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 666abdc..b664fe9 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -79,10 +79,12 @@ impl MyNoSqlData .await } + #[cfg(feature = "with-ssh")] pub async fn set_ssh_credentials(&mut self, ssh_credentials: Arc) { self.fl_url_factory.ssh_credentials = Some(ssh_credentials); } + #[cfg(feature = "with-ssh")] pub async fn set_ssh_sessions_pool(&mut self, ssh_sessions_pool: Arc) { self.fl_url_factory.ssh_sessions_pool = Some(ssh_sessions_pool); } From d8d0c9c1bd55f75647f110c4c5d717258bca89e7 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 1 Aug 2024 19:54:30 +0300 Subject: [PATCH 08/89] 0.3.3 - with-ssh --- my-no-sql-abstractions/Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 2 +- my-no-sql-data-writer/Cargo.toml | 2 +- my-no-sql-macros/Cargo.toml | 2 +- my-no-sql-sdk/Cargo.toml | 2 +- my-no-sql-tcp-reader/Cargo.toml | 2 +- my-no-sql-tcp-shared/Cargo.toml | 2 +- my-no-sql-tests/Cargo.toml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/my-no-sql-abstractions/Cargo.toml b/my-no-sql-abstractions/Cargo.toml index 472d191..240b254 100644 --- a/my-no-sql-abstractions/Cargo.toml +++ b/my-no-sql-abstractions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-abstractions" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index b773f23..40214bb 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-core" -version = "0.3.2" +version = "0.3.3" edition = "2021" [features] diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 162b9b9..10eb1eb 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-data-writer" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-macros/Cargo.toml b/my-no-sql-macros/Cargo.toml index f5c2d4e..a453887 100644 --- a/my-no-sql-macros/Cargo.toml +++ b/my-no-sql-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-macros" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index d04745a..7e9f95b 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-sdk" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index 889c407..2c52605 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-reader" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 76db413..5d92866 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-shared" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tests/Cargo.toml b/my-no-sql-tests/Cargo.toml index ba0221e..01e3770 100644 --- a/my-no-sql-tests/Cargo.toml +++ b/my-no-sql-tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tests" -version = "0.3.2" +version = "0.3.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 928bd0f04cc3e7ffa80b4439836f6a88e4c2475e Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 1 Aug 2024 19:59:35 +0300 Subject: [PATCH 09/89] Exposed with-ssh feature for data-writer --- my-no-sql-sdk/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index 7e9f95b..34c3e81 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -13,6 +13,7 @@ data-reader = ["dep:my-no-sql-tcp-reader"] tcp-contracts = ["dep:my-no-sql-tcp-shared"] master-node = ["my-no-sql-core/master-node"] debug_db_row = ["my-no-sql-core/debug_db_row"] +with-ssh = ["my-no-sql-data-writer?/with-ssh"] [dependencies] From 60d339247351be5676a84eb4386e2fac72da4f00 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 1 Aug 2024 21:43:53 +0300 Subject: [PATCH 10/89] Added Ability to connect using ssh string and SshCertCredentials --- .../my_no_sql_data_writer/fl_url_factory.rs | 22 ++++++++++++------- .../my_no_sql_data_writer.rs | 10 +++++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index af6e22c..552df22 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use flurl::FlUrl; #[cfg(feature = "with-ssh")] -use flurl::my_ssh::{SshCredentials, SshSessionsPool}; +use flurl::my_ssh::SshSessionsPool; use rust_extensions::UnsafeValue; @@ -14,9 +14,9 @@ pub struct FlUrlFactory { settings: Arc, auto_create_table_params: Option>, #[cfg(feature = "with-ssh")] - pub ssh_credentials: Option>, - #[cfg(feature = "with-ssh")] pub ssh_sessions_pool: Option>, + pub ssh_cert_credentials: + Option>, create_table_is_called: Arc>, table_name: &'static str, } @@ -30,23 +30,29 @@ impl FlUrlFactory { Self { auto_create_table_params, #[cfg(feature = "with-ssh")] - ssh_credentials: None, - #[cfg(feature = "with-ssh")] ssh_sessions_pool: None, create_table_is_called: UnsafeValue::new(false).into(), settings, table_name, + ssh_cert_credentials: None, } } - + #[cfg(not(feature = "with-ssh"))] async fn create_fl_url(&self, url: &str) -> FlUrl { let mut fl_url = flurl::FlUrl::new(url); #[cfg(feature = "with-ssh")] - if let Some(ssh_credentials) = &self.ssh_credentials { - fl_url = fl_url.set_ssh_credentials(ssh_credentials.clone()); + if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { + fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); } + fl_url + } + #[cfg(feature = "with-ssh")] + async fn create_fl_url(&self, url: &str) -> FlUrl { + let mut fl_url = + flurl::FlUrl::new_with_maybe_ssh(url, self.ssh_cert_credentials.as_ref()).await; + #[cfg(feature = "with-ssh")] if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index b664fe9..4ff323b 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -80,8 +80,14 @@ impl MyNoSqlData } #[cfg(feature = "with-ssh")] - pub async fn set_ssh_credentials(&mut self, ssh_credentials: Arc) { - self.fl_url_factory.ssh_credentials = Some(ssh_credentials); + pub async fn set_ssh_cert_credentials( + &mut self, + cert_credentials: std::collections::HashMap< + String, + flurl::my_ssh::SshCredentialsSettingsModel, + >, + ) { + self.fl_url_factory.ssh_cert_credentials = Some(cert_credentials); } #[cfg(feature = "with-ssh")] From 084f602bc893d9bde1d047bbc34ef1e7179a63c1 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 1 Aug 2024 22:03:42 +0300 Subject: [PATCH 11/89] Fixes --- .../src/my_no_sql_data_writer/fl_url_factory.rs | 11 +++-------- .../my_no_sql_data_writer/my_no_sql_data_writer.rs | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index 552df22..2654ad9 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -15,6 +15,7 @@ pub struct FlUrlFactory { auto_create_table_params: Option>, #[cfg(feature = "with-ssh")] pub ssh_sessions_pool: Option>, + #[cfg(feature = "with-ssh")] pub ssh_cert_credentials: Option>, create_table_is_called: Arc>, @@ -34,18 +35,13 @@ impl FlUrlFactory { create_table_is_called: UnsafeValue::new(false).into(), settings, table_name, + #[cfg(feature = "with-ssh")] ssh_cert_credentials: None, } } #[cfg(not(feature = "with-ssh"))] async fn create_fl_url(&self, url: &str) -> FlUrl { - let mut fl_url = flurl::FlUrl::new(url); - - #[cfg(feature = "with-ssh")] - if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { - fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); - } - + let fl_url = flurl::FlUrl::new(url); fl_url } #[cfg(feature = "with-ssh")] @@ -53,7 +49,6 @@ impl FlUrlFactory { let mut fl_url = flurl::FlUrl::new_with_maybe_ssh(url, self.ssh_cert_credentials.as_ref()).await; - #[cfg(feature = "with-ssh")] if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 4ff323b..ff8a9a8 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -80,7 +80,7 @@ impl MyNoSqlData } #[cfg(feature = "with-ssh")] - pub async fn set_ssh_cert_credentials( + pub fn set_ssh_cert_credentials( &mut self, cert_credentials: std::collections::HashMap< String, @@ -91,7 +91,7 @@ impl MyNoSqlData } #[cfg(feature = "with-ssh")] - pub async fn set_ssh_sessions_pool(&mut self, ssh_sessions_pool: Arc) { + pub fn set_ssh_sessions_pool(&mut self, ssh_sessions_pool: Arc) { self.fl_url_factory.ssh_sessions_pool = Some(ssh_sessions_pool); } From bc68c53c39b443b58688a8e6e87e4b1212393de0 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 6 Aug 2024 11:04:23 +0300 Subject: [PATCH 12/89] Fixes --- my-no-sql-data-writer/Cargo.toml | 2 +- .../src/my_no_sql_data_writer/fl_url_factory.rs | 8 ++++++++ .../src/my_no_sql_data_writer/my_no_sql_data_writer.rs | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 10eb1eb..f709488 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [features] -default = [] +default = ["with-ssh"] with-ssh = ["flurl/with-ssh"] [dependencies] diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index 2654ad9..a0bc886 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -16,6 +16,8 @@ pub struct FlUrlFactory { #[cfg(feature = "with-ssh")] pub ssh_sessions_pool: Option>, #[cfg(feature = "with-ssh")] + pub http_buffer_size: Option, + #[cfg(feature = "with-ssh")] pub ssh_cert_credentials: Option>, create_table_is_called: Arc>, @@ -37,6 +39,8 @@ impl FlUrlFactory { table_name, #[cfg(feature = "with-ssh")] ssh_cert_credentials: None, + #[cfg(feature = "with-ssh")] + http_buffer_size: None, } } #[cfg(not(feature = "with-ssh"))] @@ -53,6 +57,10 @@ impl FlUrlFactory { fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); } + if let Some(http_buffer_size) = self.http_buffer_size { + fl_url = fl_url.set_http_buffer_size(http_buffer_size); + } + fl_url } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index ff8a9a8..d52a731 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -95,6 +95,11 @@ impl MyNoSqlData self.fl_url_factory.ssh_sessions_pool = Some(ssh_sessions_pool); } + #[cfg(feature = "with-ssh")] + pub fn set_http_buffer_size(&mut self, buffer_size: usize) { + self.fl_url_factory.http_buffer_size = Some(buffer_size); + } + pub async fn create_table_if_not_exists( &self, params: &CreateTableParams, From 32963ae2c08ac155a6a65f3a23a3a792c39c8cf5 Mon Sep 17 00:00:00 2001 From: amigin Date: Mon, 12 Aug 2024 13:41:43 +0300 Subject: [PATCH 13/89] Removed with SSH --- my-no-sql-data-writer/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index f709488..10eb1eb 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [features] -default = ["with-ssh"] +default = [] with-ssh = ["flurl/with-ssh"] [dependencies] From 1a75b40ec38e05df1b973f67689baf140e709b30 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 13 Sep 2024 01:05:12 +0300 Subject: [PATCH 14/89] Made retry natively from FlUrl --- .../src/my_no_sql_data_writer/fl_url_ext.rs | 5 - .../src/my_no_sql_data_writer/with_retries.rs | 327 ++++-------------- my-no-sql-tests/src/lib.rs | 3 - my-no-sql-tests/src/test_my_no_sql_entity.rs | 3 - 4 files changed, 66 insertions(+), 272 deletions(-) delete mode 100644 my-no-sql-tests/src/test_my_no_sql_entity.rs diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs index a06a041..cd10959 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs @@ -7,7 +7,6 @@ pub trait FlUrlExt { fn with_partition_key_as_query_param(self, partition_key: &str) -> FlUrl; fn with_partition_keys_as_query_param(self, partition_keys: &[&str]) -> FlUrl; fn with_row_key_as_query_param(self, partition_key: &str) -> FlUrl; - fn with_persist_as_query_param(self, persist: bool) -> FlUrl; } impl FlUrlExt for FlUrl { @@ -44,8 +43,4 @@ impl FlUrlExt for FlUrl { fn with_row_key_as_query_param(self, row_key: &str) -> FlUrl { self.append_query_param("rowKey", Some(row_key)) } - fn with_persist_as_query_param(self, persist: bool) -> FlUrl { - let value = if persist { "1" } else { "0" }; - self.append_query_param("persist", Some(value)) - } } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs index b85dcfe..02db908 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs @@ -33,64 +33,24 @@ impl } pub async fn insert_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - - let result = super::execution::insert_entity(fl_url, entity, &self.sync_period).await; - - if result.is_ok() { - return result; - } - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::insert_entity(fl_url, entity, &self.sync_period).await } pub async fn insert_or_replace_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = - super::execution::insert_or_replace_entity(fl_url, entity, &self.sync_period).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::insert_or_replace_entity(fl_url, entity, &self.sync_period).await } pub async fn bulk_insert_or_replace( &self, entities: &[TEntity], ) -> Result<(), DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = - super::execution::bulk_insert_or_replace(fl_url, entities, &self.sync_period).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::bulk_insert_or_replace(fl_url, entities, &self.sync_period).await } pub async fn get_entity( @@ -99,27 +59,15 @@ impl row_key: &str, update_read_statistics: Option, ) -> Result, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - - let result = super::execution::get_entity( - fl_url, - partition_key, - row_key, - update_read_statistics.as_ref(), - ) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::get_entity( + fl_url, + partition_key, + row_key, + update_read_statistics.as_ref(), + ) + .await } pub async fn get_by_partition_key( @@ -127,25 +75,14 @@ impl partition_key: &str, update_read_statistics: Option, ) -> Result>, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::get_by_partition_key( - fl_url, - partition_key, - update_read_statistics.as_ref(), - ) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::get_by_partition_key( + fl_url, + partition_key, + update_read_statistics.as_ref(), + ) + .await } pub async fn get_enum_case_models_by_partition_key< @@ -159,24 +96,13 @@ impl &self, update_read_statistics: Option, ) -> Result>, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::get_enum_case_models_by_partition_key( - fl_url, - update_read_statistics.as_ref(), - ) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::get_enum_case_models_by_partition_key( + fl_url, + update_read_statistics.as_ref(), + ) + .await } pub async fn get_enum_case_model< @@ -190,42 +116,18 @@ impl &self, update_read_statistics: Option, ) -> Result, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = - super::execution::get_enum_case_model(fl_url, update_read_statistics.as_ref()) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::get_enum_case_model(fl_url, update_read_statistics.as_ref()).await } pub async fn get_by_row_key( &self, row_key: &str, ) -> Result>, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::get_by_row_key(fl_url, row_key).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::get_by_row_key(fl_url, row_key).await } pub async fn delete_enum_case< @@ -238,20 +140,9 @@ impl >( &self, ) -> Result, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::delete_enum_case(fl_url).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::delete_enum_case(fl_url).await } pub async fn delete_enum_case_with_row_key< @@ -265,20 +156,9 @@ impl &self, row_key: &str, ) -> Result, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::delete_enum_case_with_row_key(fl_url, row_key).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::delete_enum_case_with_row_key(fl_url, row_key).await } pub async fn delete_row( @@ -286,80 +166,30 @@ impl partition_key: &str, row_key: &str, ) -> Result, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::delete_row(fl_url, partition_key, row_key).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::delete_row(fl_url, partition_key, row_key).await } pub async fn delete_partitions(&self, partition_keys: &[&str]) -> Result<(), DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - - let result = - super::execution::delete_partitions(fl_url, TEntity::TABLE_NAME, partition_keys) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::delete_partitions(fl_url, TEntity::TABLE_NAME, partition_keys).await } pub async fn get_all(&self) -> Result>, DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - - let result = super::execution::get_all(fl_url).await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::get_all(fl_url).await } pub async fn clean_table_and_bulk_insert( &self, entities: &[TEntity], ) -> Result<(), DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = - super::execution::clean_table_and_bulk_insert(fl_url, entities, &self.sync_period) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::clean_table_and_bulk_insert(fl_url, entities, &self.sync_period).await } pub async fn clean_partition_and_bulk_insert( @@ -367,39 +197,14 @@ impl partition_key: &str, entities: &[TEntity], ) -> Result<(), DataWriterError> { - let mut attempt_no = 0; - loop { - let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let result = super::execution::clean_partition_and_bulk_insert( - fl_url, - partition_key, - entities, - &self.sync_period, - ) - .await; - - match result { - Ok(result) => return Ok(result), - Err(err) => { - handle_retry_error(err, attempt_no, self.max_attempts, self.attempt_delay) - .await?; - attempt_no += 1 - } - } - } - } -} - -async fn handle_retry_error( - err: DataWriterError, - attempt_no: usize, - max_attempts: usize, - attempt_delay: Duration, -) -> Result<(), DataWriterError> { - if attempt_no < max_attempts { - tokio::time::sleep(attempt_delay).await; - return Ok(()); + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + super::execution::clean_partition_and_bulk_insert( + fl_url, + partition_key, + entities, + &self.sync_period, + ) + .await } - - Err(err) } diff --git a/my-no-sql-tests/src/lib.rs b/my-no-sql-tests/src/lib.rs index 737d8ac..cd886f6 100644 --- a/my-no-sql-tests/src/lib.rs +++ b/my-no-sql-tests/src/lib.rs @@ -2,6 +2,3 @@ mod macros_tests; #[cfg(test)] mod test_new_enum_cases_added; - -#[cfg(test)] -mod test_my_no_sql_entity; diff --git a/my-no-sql-tests/src/test_my_no_sql_entity.rs b/my-no-sql-tests/src/test_my_no_sql_entity.rs deleted file mode 100644 index def473e..0000000 --- a/my-no-sql-tests/src/test_my_no_sql_entity.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub struct MyNoSqlEntity { - pub my_field: String, -} From 3c20f35326287f91a2544f5cf50f8d610c74e50f Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 20 Sep 2024 21:23:40 +0200 Subject: [PATCH 15/89] Made ability to Lazy Deserialize entities --- .../src/my_no_sql_entity.rs | 3 +- my-no-sql-core/src/entity_serializer.rs | 14 ++- .../src/my_no_sql_data_writer/execution.rs | 14 +-- my-no-sql-macros/src/entity_utils.rs | 5 +- my-no-sql-macros/src/enum_model/generate.rs | 1 + .../src/enum_of_my_no_sql_entity/generate.rs | 11 +- .../my_no_sql_entity/generate_base_impl.rs | 2 + .../src/data_reader_entities_set.rs | 56 +++++---- .../src/subscribers/callback_triggers.rs | 96 +++++++++------ .../entity_with_lazy_deserialization.rs | 83 +++++++++++++ .../get_entities_builder.rs | 10 +- .../get_entities_builder_inner.rs | 16 +-- .../get_entity_builder/get_entity_builder.rs | 9 +- .../get_entity_builder_inner.rs | 11 +- my-no-sql-tcp-reader/src/subscribers/mod.rs | 2 + .../src/subscribers/my_no_sql_data_reader.rs | 8 +- .../my_no_sql_data_reader_callbacks.rs | 27 +++-- .../my_no_sql_data_reader_callbacks_pusher.rs | 48 +++++--- .../subscribers/my_no_sql_data_reader_data.rs | 113 ++++++++++++------ .../subscribers/my_no_sql_data_reader_tcp.rs | 91 +++++++++----- .../test_enum_with_added_new_type.rs | 7 +- 21 files changed, 436 insertions(+), 191 deletions(-) create mode 100644 my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs diff --git a/my-no-sql-abstractions/src/my_no_sql_entity.rs b/my-no-sql-abstractions/src/my_no_sql_entity.rs index cb76ee2..200f661 100644 --- a/my-no-sql-abstractions/src/my_no_sql_entity.rs +++ b/my-no-sql-abstractions/src/my_no_sql_entity.rs @@ -1,5 +1,6 @@ pub trait MyNoSqlEntity { const TABLE_NAME: &'static str; + const LAZY_DESERIALIZATION: bool; fn get_partition_key(&self) -> &str; fn get_row_key(&self) -> &str; fn get_time_stamp(&self) -> i64; @@ -7,7 +8,7 @@ pub trait MyNoSqlEntity { pub trait MyNoSqlEntitySerializer: Sized { fn serialize_entity(&self) -> Vec; - fn deserialize_entity(src: &[u8]) -> Option; + fn deserialize_entity(src: &[u8]) -> Result; } pub trait GetMyNoSqlEntity { diff --git a/my-no-sql-core/src/entity_serializer.rs b/my-no-sql-core/src/entity_serializer.rs index 86dd1e1..ee2d930 100644 --- a/my-no-sql-core/src/entity_serializer.rs +++ b/my-no-sql-core/src/entity_serializer.rs @@ -12,14 +12,14 @@ where serde_json::to_vec(&entity).unwrap() } -pub fn deserialize(data: &[u8]) -> TMyNoSqlEntity +pub fn deserialize(data: &[u8]) -> Result where TMyNoSqlEntity: MyNoSqlEntity + DeserializeOwned, { let parse_result: Result = serde_json::from_slice(&data); match parse_result { - Ok(el) => return el, + Ok(el) => return Ok(el), Err(err) => { let slice_iterator = SliceIterator::new(data); let json_first_line_iterator = JsonFirstLineReader::new(slice_iterator); @@ -27,16 +27,18 @@ where match db_entity { Ok(db_entity) => { - panic!( + return Err(format!( "Table: {}. Can not parse entity with PartitionKey: [{}] and RowKey: [{}]. Err: {:?}", TMyNoSqlEntity::TABLE_NAME, db_entity.get_partition_key(data), db_entity.get_row_key(data), err - ); + )) + ; } Err(err) => { - panic!( + return Err(format!( "Table: {}. Can not extract partitionKey and rowKey. Looks like entity broken at all. Err: {:?}", TMyNoSqlEntity::TABLE_NAME, err - ) + )) + } } } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index ab86c6b..803dede 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -154,8 +154,8 @@ pub async fn get_entity &str { &self.partition_key @@ -578,9 +579,8 @@ mod tests { my_no_sql_core::entity_serializer::serialize(self) } - fn deserialize_entity(src: &[u8]) -> Option { - let result: Self = my_no_sql_core::entity_serializer::deserialize(src); - result.into() + fn deserialize_entity(src: &[u8]) -> Result { + my_no_sql_core::entity_serializer::deserialize(src) } } diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index bda750b..471c35c 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -110,9 +110,8 @@ pub fn get_fn_standard_serialize_deserialize() -> proc_macro2::TokenStream { } - fn deserialize_entity(src: &[u8]) -> Option { - let result = my_no_sql_sdk::core::entity_serializer::deserialize(src); - Some(result) + fn deserialize_entity(src: &[u8]) -> Result { + my_no_sql_sdk::core::entity_serializer::deserialize(src) } } } diff --git a/my-no-sql-macros/src/enum_model/generate.rs b/my-no-sql-macros/src/enum_model/generate.rs index 4e3f566..a75f367 100644 --- a/my-no-sql-macros/src/enum_model/generate.rs +++ b/my-no-sql-macros/src/enum_model/generate.rs @@ -50,6 +50,7 @@ pub fn generate( impl my_no_sql_sdk::abstractions::MyNoSqlEntity for #struct_name { const TABLE_NAME: &'static str = ""; + const LAZY_DESERIALIZATION: bool = true; fn get_partition_key(&self) -> &str { diff --git a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs index 5e9edb3..f67361a 100644 --- a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs @@ -50,6 +50,8 @@ pub fn generate( const TABLE_NAME: &'static str = #table_name; + const LAZY_DESERIALIZATION: bool = true; + fn get_partition_key(&self) -> &str { use my_no_sql_sdk::abstractions::MyNoSqlEntity; match self { @@ -85,7 +87,7 @@ pub fn generate( my_no_sql_sdk::core::entity_serializer::inject_partition_key_and_row_key(result, self.get_partition_key(), row_key) } - fn deserialize_entity(src: &[u8]) -> Option { + fn deserialize_entity(src: &[u8]) -> Result { #deserialize_cases } } @@ -178,12 +180,12 @@ fn get_deserialize_cases(enum_cases: &[EnumCase]) -> Result Result &str { &self.partition_key } diff --git a/my-no-sql-tcp-reader/src/data_reader_entities_set.rs b/my-no-sql-tcp-reader/src/data_reader_entities_set.rs index 549be4a..88e453d 100644 --- a/my-no-sql-tcp-reader/src/data_reader_entities_set.rs +++ b/my-no-sql-tcp-reader/src/data_reader_entities_set.rs @@ -1,15 +1,19 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; -use crate::subscribers::MyNoSqlDataReaderCallBacksPusher; +use crate::subscribers::{LazyMyNoSqlEntity, MyNoSqlDataReaderCallBacksPusher}; -pub struct DataReaderEntitiesSet { - entities: Option>>>, +pub struct DataReaderEntitiesSet< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> { + entities: Option>>>, table_name: &'static str, } -impl DataReaderEntitiesSet { +impl + DataReaderEntitiesSet +{ pub fn new(table_name: &'static str) -> Self { Self { entities: None, @@ -21,13 +25,21 @@ impl DataReaderEntitiesSe self.entities.is_some() } - pub fn as_ref(&self) -> Option<&BTreeMap>>> { + pub fn as_ref( + &self, + ) -> Option<&BTreeMap>>> { self.entities.as_ref() } + pub fn as_mut( + &mut self, + ) -> Option<&mut BTreeMap>>> { + self.entities.as_mut() + } + fn init_and_get_table( &mut self, - ) -> &mut BTreeMap>> { + ) -> &mut BTreeMap>> { if self.entities.is_none() { println!("MyNoSqlTcpReader table {} is initialized", self.table_name); self.entities = Some(BTreeMap::new()); @@ -39,9 +51,9 @@ impl DataReaderEntitiesSe pub fn init_table<'s>( &'s mut self, - data: BTreeMap>, + data: BTreeMap>>, ) -> InitTableResult<'s, TMyNoSqlEntity> { - let mut new_table: BTreeMap>> = + let mut new_table: BTreeMap>> = BTreeMap::new(); for (partition_key, src_entities_by_partition) in data { @@ -50,7 +62,6 @@ impl DataReaderEntitiesSe let by_partition = new_table.get_mut(partition_key.as_str()).unwrap(); for entity in src_entities_by_partition { - let entity = Arc::new(entity); by_partition.insert(entity.get_row_key().to_string(), entity); } } @@ -66,7 +77,7 @@ impl DataReaderEntitiesSe pub fn init_partition<'s>( &'s mut self, partition_key: &str, - src_entities: BTreeMap>, + src_entities: BTreeMap>>, ) -> InitPartitionResult<'s, TMyNoSqlEntity> { let entities = self.init_and_get_table(); @@ -76,7 +87,7 @@ impl DataReaderEntitiesSe for (row_key, entities) in src_entities { for entity in entities { - new_partition.insert(row_key.clone(), Arc::new(entity)); + new_partition.insert(row_key.clone(), entity); } } @@ -90,7 +101,7 @@ impl DataReaderEntitiesSe pub fn update_rows( &mut self, - src_data: BTreeMap>, + src_data: BTreeMap>>, callbacks: &Option>>, ) { let entities = self.init_and_get_table(); @@ -109,7 +120,6 @@ impl DataReaderEntitiesSe let by_partition = entities.get_mut(partition_key.as_str()).unwrap(); for entity in src_entities { - let entity = Arc::new(entity); if let Some(updates) = updates.as_mut() { updates.push(entity.clone()); } @@ -179,12 +189,18 @@ impl DataReaderEntitiesSe } } -pub struct InitTableResult<'s, TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static> { - pub table_now: &'s BTreeMap>>, - pub table_before: Option>>>, +pub struct InitTableResult< + 's, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> { + pub table_now: &'s BTreeMap>>, + pub table_before: Option>>>, } -pub struct InitPartitionResult<'s, TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static> { - pub partition_before: &'s BTreeMap>, - pub partition_now: Option>>, +pub struct InitPartitionResult< + 's, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> { + pub partition_before: &'s BTreeMap>, + pub partition_now: Option>>, } diff --git a/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs b/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs index bed46ad..766ee3a 100644 --- a/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs +++ b/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs @@ -1,16 +1,21 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; -use super::MyNoSqlDataReaderCallBacks; +use super::{LazyMyNoSqlEntity, MyNoSqlDataReaderCallBacks}; pub async fn trigger_table_difference< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, - before: Option>>>, - now_entities: &BTreeMap>>, + before: Option< + BTreeMap>>, + >, + now_entities: &BTreeMap< + String, + BTreeMap>, + >, ) { match before { Some(before) => { @@ -23,11 +28,14 @@ pub async fn trigger_table_difference< } pub async fn trigger_brand_new_table< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, - now_entities: &BTreeMap>>, + now_entities: &BTreeMap< + String, + BTreeMap>, + >, ) { for (partition_key, now_partition) in now_entities { let mut added_entities = Vec::new(); @@ -44,12 +52,15 @@ pub async fn trigger_brand_new_table< } pub async fn trigger_old_and_new_table_difference< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, - mut before: BTreeMap>>, - now_entities: &BTreeMap>>, + mut before: BTreeMap>>, + now_entities: &BTreeMap< + String, + BTreeMap>, + >, ) { for (now_partition_key, now_partition) in now_entities { let before_partition = before.remove(now_partition_key); @@ -79,13 +90,13 @@ pub async fn trigger_old_and_new_table_difference< } pub async fn trigger_partition_difference< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, partition_key: &str, - before_partition: Option>>, - now_partition: &BTreeMap>, + before_partition: Option>>, + now_partition: &BTreeMap>, ) { match before_partition { Some(mut before_partition) => { @@ -125,12 +136,12 @@ pub async fn trigger_partition_difference< } pub async fn trigger_brand_new_partition< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, partition_key: &str, - partition: &BTreeMap>, + partition: &BTreeMap>, ) { let mut inserted_or_replaced = Vec::new(); for entity in partition.values() { @@ -146,17 +157,18 @@ pub async fn trigger_brand_new_partition< #[cfg(test)] mod tests { - use std::{collections::BTreeMap, sync::Arc}; + use std::collections::BTreeMap; use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use serde_derive::{Deserialize, Serialize}; use tokio::sync::Mutex; - use crate::subscribers::MyNoSqlDataReaderCallBacks; + use crate::subscribers::{LazyMyNoSqlEntity, MyNoSqlDataReaderCallBacks}; struct TestCallbacksInner { - inserted_or_replaced_entities: BTreeMap>>, - deleted: BTreeMap>>, + inserted_or_replaced_entities: + BTreeMap>>, + deleted: BTreeMap>>, } pub struct TestCallbacks { @@ -176,7 +188,11 @@ mod tests { #[async_trait::async_trait] impl MyNoSqlDataReaderCallBacks for TestCallbacks { - async fn inserted_or_replaced(&self, partition_key: &str, entities: Vec>) { + async fn inserted_or_replaced( + &self, + partition_key: &str, + entities: Vec>, + ) { let mut write_access = self.data.lock().await; match write_access .inserted_or_replaced_entities @@ -194,7 +210,11 @@ mod tests { } } - async fn deleted(&self, partition_key: &str, entities: Vec>) { + async fn deleted( + &self, + partition_key: &str, + entities: Vec>, + ) { let mut write_access = self.data.lock().await; match write_access.deleted.get_mut(partition_key) { Some(db_partition) => { @@ -229,6 +249,7 @@ mod tests { impl MyNoSqlEntity for TestRow { const TABLE_NAME: &'static str = "Test"; + const LAZY_DESERIALIZATION: bool = false; fn get_partition_key(&self) -> &str { self.partition_key.as_str() @@ -246,9 +267,8 @@ mod tests { my_no_sql_core::entity_serializer::serialize(self) } - fn deserialize_entity(src: &[u8]) -> Option { - let result = my_no_sql_core::entity_serializer::deserialize(src); - Some(result) + fn deserialize_entity(src: &[u8]) -> Result { + my_no_sql_core::entity_serializer::deserialize(src) } } @@ -256,15 +276,16 @@ mod tests { pub async fn test_we_had_data_in_table_and_new_table_is_empty() { let test_callback = TestCallbacks::new(); - let mut before_rows = BTreeMap::new(); + let mut before_rows: BTreeMap> = + BTreeMap::new(); before_rows.insert( "RK1".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK1".to_string(), 1)), + TestRow::new("PK1".to_string(), "RK1".to_string(), 1).into(), ); before_rows.insert( "RK2".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK2".to_string(), 1)), + TestRow::new("PK1".to_string(), "RK2".to_string(), 1).into(), ); let mut before = BTreeMap::new(); @@ -284,15 +305,16 @@ mod tests { pub async fn test_brand_new_table() { let test_callback = TestCallbacks::new(); - let mut after_rows = BTreeMap::new(); + let mut after_rows: BTreeMap> = + BTreeMap::new(); after_rows.insert( "RK1".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK1".to_string(), 1)), + TestRow::new("PK1".to_string(), "RK1".to_string(), 1).into(), ); after_rows.insert( "RK2".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK2".to_string(), 1)), + TestRow::new("PK1".to_string(), "RK2".to_string(), 1).into(), ); let mut after = BTreeMap::new(); @@ -316,24 +338,26 @@ mod tests { pub async fn test_we_have_updates_in_table() { let test_callback = TestCallbacks::new(); - let mut before_partition = BTreeMap::new(); + let mut before_partition: BTreeMap> = + BTreeMap::new(); before_partition.insert( "RK1".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK1".to_string(), 1)), + TestRow::new("PK1".to_string(), "RK1".to_string(), 1).into(), ); before_partition.insert( "RK2".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK2".to_string(), 1)), + TestRow::new("PK1".to_string(), "RK2".to_string(), 1).into(), ); let mut before = BTreeMap::new(); before.insert("PK1".to_string(), before_partition); - let mut after_partition = BTreeMap::new(); + let mut after_partition: BTreeMap> = + BTreeMap::new(); after_partition.insert( "RK2".to_string(), - Arc::new(TestRow::new("PK1".to_string(), "RK2".to_string(), 2)), + TestRow::new("PK1".to_string(), "RK2".to_string(), 2).into(), ); let mut after = BTreeMap::new(); diff --git a/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs b/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs new file mode 100644 index 0000000..bae980d --- /dev/null +++ b/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs @@ -0,0 +1,83 @@ +use std::{fmt::Debug, sync::Arc}; + +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; +use my_no_sql_core::db_json_entity::DbJsonEntity; + +pub struct EntityRawData { + pub db_json_entity: DbJsonEntity, + pub data: Vec, +} + +pub enum LazyMyNoSqlEntity< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> { + Raw(Arc), + Deserialized(Arc), +} + +impl + LazyMyNoSqlEntity +{ + pub fn get_partition_key(&self) -> &str { + match self { + LazyMyNoSqlEntity::Deserialized(entity) => entity.get_partition_key(), + LazyMyNoSqlEntity::Raw(src) => src.db_json_entity.get_partition_key(&src.data), + } + } + + pub fn get_row_key(&self) -> &str { + match self { + LazyMyNoSqlEntity::Deserialized(entity) => entity.get_row_key(), + LazyMyNoSqlEntity::Raw(src) => src.db_json_entity.get_row_key(&src.data), + } + } + + pub fn get(&mut self) -> Arc { + match self { + LazyMyNoSqlEntity::Deserialized(entity) => entity.clone(), + LazyMyNoSqlEntity::Raw(src) => { + let entity = TMyNoSqlEntity::deserialize_entity(&src.data).unwrap(); + let entity = Arc::new(entity); + *self = LazyMyNoSqlEntity::Deserialized(entity.clone()); + entity + } + } + } + + pub fn clone(&self) -> Self { + match self { + LazyMyNoSqlEntity::Deserialized(entity) => { + LazyMyNoSqlEntity::Deserialized(entity.clone()) + } + LazyMyNoSqlEntity::Raw(src) => LazyMyNoSqlEntity::Raw(src.clone()), + } + } +} + +impl + From for LazyMyNoSqlEntity +{ + fn from(value: TMyNoSqlEntity) -> Self { + LazyMyNoSqlEntity::Deserialized(Arc::new(value)) + } +} + +impl + std::fmt::Debug for LazyMyNoSqlEntity +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LazyMyNoSqlEntity::Deserialized(entity) => write!(f, "Deserialized({:?})", entity), + LazyMyNoSqlEntity::Raw(data) => { + write!( + f, + "Raw(PartitionKey: {}, RowKey: {}, Timestamp:{:?}, DataSize: {})", + self.get_partition_key(), + self.get_row_key(), + data.db_json_entity.get_time_stamp(data.data.as_slice()), + data.data.len() + ) + } + } + } +} diff --git a/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder.rs b/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder.rs index 8b2233e..a22b32c 100644 --- a/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder.rs +++ b/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::date_time::DateTimeAsMicroseconds; use super::{super::my_no_sql_data_reader_tcp::MyNoSqlDataReaderInner, GetEntitiesBuilderInner}; @@ -8,13 +8,17 @@ use super::{super::my_no_sql_data_reader_tcp::MyNoSqlDataReaderInner, GetEntitie #[cfg(feature = "mocks")] use super::GetEntitiesBuilderMock; -pub enum GetEntitiesBuilder { +pub enum GetEntitiesBuilder< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { Inner(GetEntitiesBuilderInner), #[cfg(feature = "mocks")] Mock(GetEntitiesBuilderMock), } -impl GetEntitiesBuilder { +impl + GetEntitiesBuilder +{ pub fn new(partition_key: String, inner: Arc>) -> Self { Self::Inner(GetEntitiesBuilderInner::new(partition_key, inner)) } diff --git a/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_inner.rs b/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_inner.rs index d54d29c..ebe1074 100644 --- a/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_inner.rs +++ b/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_inner.rs @@ -1,18 +1,20 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::UpdateEntityStatisticsData; use rust_extensions::date_time::DateTimeAsMicroseconds; use super::super::my_no_sql_data_reader_tcp::MyNoSqlDataReaderInner; -pub struct GetEntitiesBuilderInner { +pub struct GetEntitiesBuilderInner< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { partition_key: String, update_statistic_data: UpdateEntityStatisticsData, inner: Arc>, } -impl +impl GetEntitiesBuilderInner { pub fn new(partition_key: String, inner: Arc>) -> Self { @@ -41,7 +43,7 @@ impl pub async fn get_as_vec(&self) -> Option>> { let db_rows = { - let reader = self.inner.get_data().read().await; + let mut reader = self.inner.get_data().lock().await; reader.get_by_partition_as_vec(self.partition_key.as_str()) }?; @@ -63,7 +65,7 @@ impl filter: impl Fn(&TMyNoSqlEntity) -> bool, ) -> Option>> { let db_rows = { - let reader = self.inner.get_data().read().await; + let mut reader = self.inner.get_data().lock().await; reader.get_by_partition_as_vec_with_filter(&self.partition_key, filter) }?; @@ -82,7 +84,7 @@ impl pub async fn get_as_btree_map(&self) -> Option>> { let db_rows = { - let reader = self.inner.get_data().read().await; + let mut reader = self.inner.get_data().lock().await; reader.get_by_partition(&self.partition_key) }?; @@ -104,7 +106,7 @@ impl filter: impl Fn(&TMyNoSqlEntity) -> bool, ) -> Option>> { let db_rows = { - let reader = self.inner.get_data().read().await; + let mut reader = self.inner.get_data().lock().await; reader.get_by_partition_with_filter(&self.partition_key, filter) }?; diff --git a/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder.rs b/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder.rs index 598bb42..40b8072 100644 --- a/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder.rs +++ b/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder.rs @@ -1,18 +1,21 @@ use std::sync::Arc; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::date_time::DateTimeAsMicroseconds; #[cfg(feature = "mocks")] use super::GetEntityBuilderMock; use super::{super::my_no_sql_data_reader_tcp::MyNoSqlDataReaderInner, GetEntityBuilderInner}; -pub enum GetEntityBuilder<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> { +pub enum GetEntityBuilder< + 's, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { Inner(GetEntityBuilderInner<'s, TMyNoSqlEntity>), #[cfg(feature = "mocks")] Mock(GetEntityBuilderMock<'s, TMyNoSqlEntity>), } -impl<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> +impl<'s, TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static> GetEntityBuilder<'s, TMyNoSqlEntity> { pub fn new( diff --git a/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_inner.rs b/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_inner.rs index b68f4a7..91ef6a4 100644 --- a/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_inner.rs +++ b/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_inner.rs @@ -1,19 +1,22 @@ use std::sync::Arc; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::UpdateEntityStatisticsData; use rust_extensions::date_time::DateTimeAsMicroseconds; use super::super::my_no_sql_data_reader_tcp::MyNoSqlDataReaderInner; -pub struct GetEntityBuilderInner<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> { +pub struct GetEntityBuilderInner< + 's, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { partition_key: &'s str, row_key: &'s str, update_statistic_data: UpdateEntityStatisticsData, inner: Arc>, } -impl<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> +impl<'s, TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static> GetEntityBuilderInner<'s, TMyNoSqlEntity> { pub fn new( @@ -47,7 +50,7 @@ impl<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> pub async fn execute(&self) -> Option> { let result = { - let reader = self.inner.get_data().read().await; + let mut reader = self.inner.get_data().lock().await; reader.get_entity(self.partition_key, self.row_key) }; diff --git a/my-no-sql-tcp-reader/src/subscribers/mod.rs b/my-no-sql-tcp-reader/src/subscribers/mod.rs index 765aaf1..0fabee5 100644 --- a/my-no-sql-tcp-reader/src/subscribers/mod.rs +++ b/my-no-sql-tcp-reader/src/subscribers/mod.rs @@ -26,3 +26,5 @@ pub use my_no_sql_data_reader_mock::*; mod my_no_sql_data_reader_mock_inner; #[cfg(feature = "mocks")] pub use my_no_sql_data_reader_mock_inner::*; +mod entity_with_lazy_deserialization; +pub use entity_with_lazy_deserialization::*; diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs index 7760ec5..d38c3bf 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs @@ -1,13 +1,16 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use crate::MyNoSqlDataReaderCallBacks; use super::{GetEntitiesBuilder, GetEntityBuilder}; #[async_trait::async_trait] -pub trait MyNoSqlDataReader { +pub trait MyNoSqlDataReader< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> +{ async fn get_table_snapshot_as_vec(&self) -> Option>>; async fn get_by_partition_key( @@ -61,6 +64,7 @@ pub trait MyNoSqlDataReader> + Sync diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks.rs index 9921243..420c291 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks.rs @@ -1,26 +1,37 @@ -use std::sync::Arc; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; -use my_no_sql_abstractions::MyNoSqlEntity; +use super::LazyMyNoSqlEntity; #[async_trait::async_trait] -pub trait MyNoSqlDataReaderCallBacks { - async fn inserted_or_replaced(&self, partition_key: &str, entities: Vec>); - async fn deleted(&self, partition_key: &str, entities: Vec>); +pub trait MyNoSqlDataReaderCallBacks< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> +{ + async fn inserted_or_replaced( + &self, + partition_key: &str, + entities: Vec>, + ); + async fn deleted(&self, partition_key: &str, entities: Vec>); } #[async_trait::async_trait] -impl +impl MyNoSqlDataReaderCallBacks for () { async fn inserted_or_replaced( &self, _partition_key: &str, - _entities: Vec>, + _entities: Vec>, ) { panic!("This is a dumb implementation") } - async fn deleted(&self, _partition_key: &str, _entities: Vec>) { + async fn deleted( + &self, + _partition_key: &str, + _entities: Vec>, + ) { panic!("This is a dumb implementation") } } diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks_pusher.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks_pusher.rs index e260fda..002be1e 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks_pusher.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_callbacks_pusher.rs @@ -1,28 +1,30 @@ use std::sync::Arc; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::{ events_loop::{EventsLoop, EventsLoopTick}, ApplicationStates, }; -use super::MyNoSqlDataReaderCallBacks; +use super::{LazyMyNoSqlEntity, MyNoSqlDataReaderCallBacks}; -pub enum PusherEvents { - InsertedOrReplaced(String, Vec>), - Deleted(String, Vec>), +pub enum PusherEvents< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> { + InsertedOrReplaced(String, Vec>), + Deleted(String, Vec>), } pub struct MyNoSqlDataReaderCallBacksPusher where - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, { events_loop: EventsLoop>, } impl MyNoSqlDataReaderCallBacksPusher where - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, { pub async fn new< TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks + Send + Sync + 'static, @@ -40,14 +42,22 @@ where Self { events_loop } } - pub fn inserted_or_replaced(&self, partition_key: &str, entities: Vec>) { + pub fn inserted_or_replaced( + &self, + partition_key: &str, + entities: Vec>, + ) { self.events_loop.send(PusherEvents::InsertedOrReplaced( partition_key.to_string(), entities, )); } - pub fn deleted(&self, partition_key: &str, entities: Vec>) { + pub fn deleted( + &self, + partition_key: &str, + entities: Vec>, + ) { self.events_loop .send(PusherEvents::Deleted(partition_key.to_string(), entities)); } @@ -57,23 +67,31 @@ where impl MyNoSqlDataReaderCallBacks for MyNoSqlDataReaderCallBacksPusher where - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, { - async fn inserted_or_replaced(&self, partition_key: &str, entities: Vec>) { + async fn inserted_or_replaced( + &self, + partition_key: &str, + entities: Vec>, + ) { self.events_loop.send(PusherEvents::InsertedOrReplaced( partition_key.to_string(), entities, )); } - async fn deleted(&self, partition_key: &str, entities: Vec>) { + async fn deleted( + &self, + partition_key: &str, + entities: Vec>, + ) { self.events_loop .send(PusherEvents::Deleted(partition_key.to_string(), entities)); } } pub struct MyNoSqlDataReaderCallBacksSender< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, > { callbacks: Arc, @@ -81,7 +99,7 @@ pub struct MyNoSqlDataReaderCallBacksSender< } impl< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks + Send + Sync + 'static, > MyNoSqlDataReaderCallBacksSender { @@ -92,7 +110,7 @@ impl< #[async_trait::async_trait] impl< - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks + Send + Sync + 'static, > EventsLoopTick> for MyNoSqlDataReaderCallBacksSender diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs index 0c3ebfb..cb23ff5 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs @@ -1,13 +1,15 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::ApplicationStates; use crate::DataReaderEntitiesSet; -use super::{MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderCallBacksPusher}; +use super::{LazyMyNoSqlEntity, MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderCallBacksPusher}; -pub struct MyNoSqlDataReaderData { +pub struct MyNoSqlDataReaderData< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, +> { entities: DataReaderEntitiesSet, callbacks: Option>>, app_states: Arc, @@ -15,7 +17,7 @@ pub struct MyNoSqlDataReaderData MyNoSqlDataReaderData where - TMyNoSqlEntity: MyNoSqlEntity + Send + Sync + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Send + Sync + 'static, { pub async fn new( table_name: &'static str, @@ -40,7 +42,10 @@ where self.callbacks = Some(Arc::new(pusher)); } - pub async fn init_table(&mut self, data: BTreeMap>) { + pub async fn init_table( + &mut self, + data: BTreeMap>>, + ) { let init_table_result = self.entities.init_table(data); if let Some(callbacks) = self.callbacks.as_ref() { @@ -56,7 +61,7 @@ where pub async fn init_partition( &mut self, partition_key: &str, - src_entities: BTreeMap>, + src_entities: BTreeMap>>, ) { //let callbacks = self.callbacks.clone(); @@ -73,7 +78,10 @@ where } } - pub fn update_rows(&mut self, src_data: BTreeMap>) { + pub fn update_rows( + &mut self, + src_data: BTreeMap>>, + ) { self.entities.update_rows(src_data, &self.callbacks); } @@ -82,15 +90,29 @@ where } pub fn get_table_snapshot( - &self, + &mut self, ) -> Option>>> { - let entities = self.entities.as_ref()?; + let entities = self.entities.as_mut()?; + if entities.len() == 0 { + return None; + } + + let mut result = BTreeMap::new(); + for (partition_key, entities) in entities.iter_mut() { + let mut to_insert = BTreeMap::new(); + + for (row_key, entity) in entities.iter_mut() { + to_insert.insert(row_key.clone(), entity.get()); + } + + result.insert(partition_key.clone(), to_insert); + } - return Some(entities.clone()); + return Some(result); } - pub fn get_table_snapshot_as_vec(&self) -> Option>> { - let entities = self.entities.as_ref()?; + pub fn get_table_snapshot_as_vec(&mut self) -> Option>> { + let entities = self.entities.as_mut()?; if entities.len() == 0 { return None; @@ -98,49 +120,60 @@ where let mut result = Vec::new(); - for partition in entities.values() { - for entity in partition.values() { - result.push(entity.clone()); + for partition in entities.values_mut() { + for entity in partition.values_mut() { + result.push(entity.get()); } } Some(result) } - pub fn get_entity(&self, partition_key: &str, row_key: &str) -> Option> { - let entities = self.entities.as_ref()?; + pub fn get_entity( + &mut self, + partition_key: &str, + row_key: &str, + ) -> Option> { + let entities = self.entities.as_mut()?; - let partition = entities.get(partition_key)?; + let partition = entities.get_mut(partition_key)?; - let row = partition.get(row_key)?; + let row = partition.get_mut(row_key)?; - Some(row.clone()) + Some(row.get()) } pub fn get_by_partition( - &self, + &mut self, partition_key: &str, ) -> Option>> { - let entities = self.entities.as_ref()?; + let entities = self.entities.as_mut()?; + + let partition = entities.get_mut(partition_key)?; - let partition = entities.get(partition_key)?; + let mut result = BTreeMap::new(); + + for itm in partition.iter_mut() { + result.insert(itm.0.clone(), itm.1.get()); + } - Some(partition.clone()) + Some(result) } pub fn get_by_partition_with_filter( - &self, + &mut self, partition_key: &str, filter: impl Fn(&TMyNoSqlEntity) -> bool, ) -> Option>> { - let entities = self.entities.as_ref()?; + let entities = self.entities.as_mut()?; - let partition = entities.get(partition_key)?; + let partition = entities.get_mut(partition_key)?; let mut result = BTreeMap::new(); - for db_row in partition.values() { - if filter(db_row) { + for db_row in partition.values_mut() { + let db_row = db_row.get(); + if filter(&db_row) { result.insert(db_row.get_row_key().to_string(), db_row.clone()); } } @@ -160,10 +193,13 @@ where entities.contains_key(partition_key) } - pub fn get_by_partition_as_vec(&self, partition_key: &str) -> Option>> { - let entities = self.entities.as_ref()?; + pub fn get_by_partition_as_vec( + &mut self, + partition_key: &str, + ) -> Option>> { + let entities = self.entities.as_mut()?; - let partition = entities.get(partition_key)?; + let partition = entities.get_mut(partition_key)?; if partition.len() == 0 { return None; @@ -171,21 +207,21 @@ where let mut result = Vec::with_capacity(partition.len()); - for db_row in partition.values() { - result.push(db_row.clone()); + for db_row in partition.values_mut() { + result.push(db_row.get()); } Some(result) } pub fn get_by_partition_as_vec_with_filter( - &self, + &mut self, partition_key: &str, filter: impl Fn(&TMyNoSqlEntity) -> bool, ) -> Option>> { - let entities = self.entities.as_ref()?; + let entities = self.entities.as_mut()?; - let partition = entities.get(partition_key)?; + let partition = entities.get_mut(partition_key)?; if partition.len() == 0 { return None; @@ -193,7 +229,8 @@ where let mut result = Vec::with_capacity(partition.len()); - for db_row in partition.values() { + for db_row in partition.values_mut() { + let db_row = db_row.get(); if filter(db_row.as_ref()) { result.push(db_row.clone()); } diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 733c995..82c13a5 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -6,20 +6,24 @@ use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::SyncToMainNodeHandler; use rust_extensions::{array_of_bytes_iterator::SliceIterator, ApplicationStates, StrOrString}; use serde::de::DeserializeOwned; -use tokio::sync::RwLock; +use tokio::sync::Mutex; use super::{ - GetEntitiesBuilder, GetEntityBuilder, MyNoSqlDataReader, MyNoSqlDataReaderCallBacks, - MyNoSqlDataReaderData, UpdateEvent, + EntityRawData, GetEntitiesBuilder, GetEntityBuilder, LazyMyNoSqlEntity, MyNoSqlDataReader, + MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderData, UpdateEvent, }; -pub struct MyNoSqlDataReaderInner { - data: RwLock>, +pub struct MyNoSqlDataReaderInner< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { + data: Mutex>, sync_handler: Arc, } -impl MyNoSqlDataReaderInner { - pub fn get_data(&self) -> &RwLock> { +impl + MyNoSqlDataReaderInner +{ + pub fn get_data(&self) -> &Mutex> { &self.data } @@ -28,7 +32,9 @@ impl MyNoSqlDataReaderInn } } -pub struct MyNoSqlDataReaderTcp { +pub struct MyNoSqlDataReaderTcp< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { inner: Arc>, } @@ -42,7 +48,7 @@ where ) -> Self { Self { inner: Arc::new(MyNoSqlDataReaderInner { - data: RwLock::new( + data: Mutex::new( MyNoSqlDataReaderData::new(TMyNoSqlEntity::TABLE_NAME, app_states).await, ), sync_handler, @@ -53,12 +59,12 @@ where pub async fn get_table_snapshot( &self, ) -> Option>>> { - let reader = self.inner.data.read().await; + let mut reader = self.inner.data.lock().await; return reader.get_table_snapshot(); } pub async fn get_table_snapshot_as_vec(&self) -> Option>> { - let reader = self.inner.data.read().await; + let mut reader = self.inner.data.lock().await; reader.get_table_snapshot_as_vec() } @@ -66,7 +72,7 @@ where &self, partition_key: &str, ) -> Option>> { - let reader = self.inner.data.read().await; + let mut reader = self.inner.data.lock().await; reader.get_by_partition(partition_key) } @@ -74,7 +80,7 @@ where &self, partition_key: &str, ) -> Option>> { - let reader = self.inner.data.read().await; + let mut reader = self.inner.data.lock().await; reader.get_by_partition_as_vec(partition_key) } @@ -83,7 +89,7 @@ where partition_key: &str, row_key: &str, ) -> Option> { - let reader = self.inner.data.read().await; + let mut reader = self.inner.data.lock().await; reader.get_entity(partition_key, row_key) } @@ -103,12 +109,14 @@ where } pub async fn has_partition(&self, partition_key: &str) -> bool { - let reader: tokio::sync::RwLockReadGuard<'_, MyNoSqlDataReaderData> = - self.inner.data.read().await; + let reader = self.inner.data.lock().await; reader.has_partition(partition_key) } - pub fn deserialize_array(&self, data: &[u8]) -> BTreeMap> { + pub fn deserialize_array( + &self, + data: &[u8], + ) -> BTreeMap>> { let slice_iterator = SliceIterator::new(data); let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); @@ -126,17 +134,34 @@ where let db_entity_data = db_entity.unwrap(); - let el = - TMyNoSqlEntity::deserialize_entity(db_entity_data.as_bytes(&json_array_iterator)); - - if let Some(el) = el { - let partition_key = el.get_partition_key(); - if !result.contains_key(partition_key) { - result.insert(partition_key.to_string(), Vec::new()); - } - - result.get_mut(partition_key).unwrap().push(el); + let item_to_insert = if TMyNoSqlEntity::LAZY_DESERIALIZATION { + let data = db_entity_data.as_bytes(&json_array_iterator).to_vec(); + let db_json_entity = + my_no_sql_core::db_json_entity::DbJsonEntity::from_slice(&data).unwrap(); + + LazyMyNoSqlEntity::Raw( + EntityRawData { + db_json_entity, + data, + } + .into(), + ) + } else { + LazyMyNoSqlEntity::Deserialized( + TMyNoSqlEntity::deserialize_entity( + db_entity_data.as_bytes(&json_array_iterator), + ) + .unwrap() + .into(), + ) + }; + + let partition_key = item_to_insert.get_partition_key(); + if !result.contains_key(partition_key) { + result.insert(partition_key.to_string(), Vec::new()); } + + result.get_mut(partition_key).unwrap().push(item_to_insert); } result @@ -221,26 +246,26 @@ impl Upda async fn init_table(&self, data: Vec) { let data = self.deserialize_array(data.as_slice()); - let mut write_access = self.inner.data.write().await; + let mut write_access = self.inner.data.lock().await; write_access.init_table(data).await; } async fn init_partition(&self, partition_key: &str, data: Vec) { let data = self.deserialize_array(data.as_slice()); - let mut write_access = self.inner.data.write().await; + let mut write_access = self.inner.data.lock().await; write_access.init_partition(partition_key, data).await; } async fn update_rows(&self, data: Vec) { let data = self.deserialize_array(data.as_slice()); - let mut write_access = self.inner.data.write().await; + let mut write_access = self.inner.data.lock().await; write_access.update_rows(data); } async fn delete_rows(&self, rows_to_delete: Vec) { - let mut write_access = self.inner.data.write().await; + let mut write_access = self.inner.data.lock().await; write_access.delete_rows(rows_to_delete); } } @@ -292,7 +317,7 @@ where async fn wait_until_first_data_arrives(&self) { loop { { - let reader = self.inner.data.read().await; + let reader = self.inner.data.lock().await; if reader.has_entities_at_all().await { return; } @@ -308,7 +333,7 @@ where &self, callbacks: Arc, ) { - let mut write_access = self.inner.data.write().await; + let mut write_access = self.inner.data.lock().await; write_access.assign_callback(callbacks).await; } } diff --git a/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs b/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs index fe18c1b..0663b46 100644 --- a/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs +++ b/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs @@ -1,4 +1,5 @@ use my_no_sql_macros::*; + use serde::*; #[enum_of_my_no_sql_entity(table_name:"Test", generate_unwraps)] @@ -45,6 +46,10 @@ mod tests { let model_ver1 = MyNoSqlEnumEntityTestVer1::deserialize_entity(result.as_slice()); - assert!(model_ver1.is_none()) + assert!(model_ver1.is_err()); + + if let Err(err) = model_ver1 { + println!("{}", err); + } } } From 7341d278657a420b1e5bf7c5053860e2713b5dba Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 20 Sep 2024 21:30:27 +0200 Subject: [PATCH 16/89] Made tests compilable --- .../src/db/db_partition/db_rows_container.rs | 35 +++++++++++-------- .../src/db/db_row/test_expires_update.rs | 30 +++++++++------- .../src/db/db_table/db_table_master_node.rs | 6 ++-- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/my-no-sql-core/src/db/db_partition/db_rows_container.rs b/my-no-sql-core/src/db/db_partition/db_rows_container.rs index 30673d4..9830433 100644 --- a/my-no-sql-core/src/db/db_partition/db_rows_container.rs +++ b/my-no-sql-core/src/db/db_partition/db_rows_container.rs @@ -169,7 +169,8 @@ mod tests { }"#; let time_stamp = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &time_stamp).unwrap(); + let db_row = + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &time_stamp).unwrap(); let mut db_rows = DbRowsContainer::new(); @@ -186,7 +187,8 @@ mod tests { }"#; let time_stamp = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &time_stamp).unwrap(); + let db_row = + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &time_stamp).unwrap(); let mut db_rows = DbRowsContainer::new(); @@ -203,7 +205,8 @@ mod tests { "Expires": "2019-01-01T00:00:00" }"#; let time_stamp = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &time_stamp).unwrap(); + let db_row = + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &time_stamp).unwrap(); let mut db_rows = DbRowsContainer::new(); @@ -223,7 +226,8 @@ mod tests { let time_stamp = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &time_stamp).unwrap(); + let db_row = + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &time_stamp).unwrap(); let mut db_rows = DbRowsContainer::new(); @@ -253,7 +257,8 @@ mod tests { }"#; let time_stamp = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &time_stamp).unwrap(); + let db_row = + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &time_stamp).unwrap(); let mut db_rows = DbRowsContainer::new(); @@ -291,7 +296,7 @@ mod tests { }"#; let now = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &now).unwrap(); + let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); db_rows.insert(Arc::new(db_row)); @@ -321,7 +326,7 @@ mod tests { let now = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &now).unwrap(); + let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); db_rows.insert(Arc::new(db_row)); @@ -346,7 +351,7 @@ mod tests { }"#; let now = JsonTimeStamp::now(); - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &now).unwrap(); + let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); db_rows.insert(Arc::new(db_row)); @@ -370,9 +375,11 @@ mod tests { "RowKey": "test1", }"#; - let db_row = - DbJsonEntity::parse_into_db_row(json.as_bytes(), &JsonTimeStamp::from_date_time(now)) - .unwrap(); + let db_row = DbJsonEntity::parse_into_db_row( + json.as_bytes().into(), + &JsonTimeStamp::from_date_time(now), + ) + .unwrap(); db_rows.insert(Arc::new(db_row)); @@ -386,7 +393,7 @@ mod tests { }"#; let db_row = DbJsonEntity::parse_into_db_row( - raw_json.as_bytes(), + raw_json.as_bytes().into(), &JsonTimeStamp::from_date_time(now), ) .unwrap(); @@ -403,7 +410,7 @@ mod tests { }"#; let db_row = DbJsonEntity::parse_into_db_row( - json_db_row.as_bytes(), + json_db_row.as_bytes().into(), &JsonTimeStamp::from_date_time(now), ) .unwrap(); @@ -420,7 +427,7 @@ mod tests { }"#; let db_row = DbJsonEntity::parse_into_db_row( - raw_json.as_bytes(), + raw_json.as_bytes().into(), &JsonTimeStamp::from_date_time(now), ) .unwrap(); diff --git a/my-no-sql-core/src/db/db_row/test_expires_update.rs b/my-no-sql-core/src/db/db_row/test_expires_update.rs index c446a49..1e96589 100644 --- a/my-no-sql-core/src/db/db_row/test_expires_update.rs +++ b/my-no-sql-core/src/db/db_row/test_expires_update.rs @@ -15,7 +15,8 @@ mod test { let inject_time_stamp = JsonTimeStamp::now(); let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &inject_time_stamp) + .unwrap(); let new_expires = DateTimeAsMicroseconds::from_str("2020-01-02T01:02:03").unwrap(); @@ -25,7 +26,7 @@ mod test { db_row.write_json(&mut result_json); - let result_entity = DbJsonEntity::new(&result_json).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); assert_eq!(result_entity.get_partition_key(&result_json), "TestPk"); assert_eq!(result_entity.get_row_key(&result_json), "TestRk"); @@ -45,7 +46,8 @@ mod test { let inject_time_stamp = JsonTimeStamp::now(); let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &inject_time_stamp) + .unwrap(); let new_expires = DateTimeAsMicroseconds::from_str("2020-01-02T01:02:03").unwrap(); @@ -55,7 +57,7 @@ mod test { db_row.write_json(&mut result_json); - let result_entity = DbJsonEntity::new(&result_json).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); assert_eq!(result_entity.get_row_key(&result_json), "Rk"); @@ -73,7 +75,8 @@ mod test { let inject_time_stamp = JsonTimeStamp::now(); let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &inject_time_stamp) + .unwrap(); db_row.update_expires(None); @@ -83,7 +86,7 @@ mod test { println!("Result: {}", std::str::from_utf8(&result_json).unwrap()); - let result_entity = DbJsonEntity::new(&result_json).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); assert_eq!(result_entity.get_row_key(&result_json), "Rk"); @@ -97,7 +100,8 @@ mod test { let inject_time_stamp = JsonTimeStamp::now(); let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &inject_time_stamp) + .unwrap(); db_row.update_expires(None); @@ -107,7 +111,7 @@ mod test { println!("Result: {}", std::str::from_utf8(&result_json).unwrap()); - let result_entity = DbJsonEntity::new(&result_json).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); assert_eq!(result_entity.get_row_key(&result_json), "Rk"); @@ -121,7 +125,8 @@ mod test { let inject_time_stamp = JsonTimeStamp::now(); let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &inject_time_stamp) + .unwrap(); db_row.update_expires(None); @@ -131,7 +136,7 @@ mod test { println!("Result: {}", std::str::from_utf8(&result_json).unwrap()); - let result_entity = DbJsonEntity::new(&result_json).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); assert_eq!(result_entity.get_row_key(&result_json), "Rk"); @@ -148,7 +153,8 @@ mod test { let inject_time_stamp = JsonTimeStamp::now(); let db_row = - DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &inject_time_stamp).unwrap(); + DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &inject_time_stamp) + .unwrap(); db_row.update_expires(None); @@ -162,7 +168,7 @@ mod test { result_json.len() ); - let result_entity = DbJsonEntity::new(&result_json).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); assert_eq!(result_entity.get_row_key(&result_json), "Rk"); diff --git a/my-no-sql-core/src/db/db_table/db_table_master_node.rs b/my-no-sql-core/src/db/db_table/db_table_master_node.rs index 45fe25d..0912393 100644 --- a/my-no-sql-core/src/db/db_table/db_table_master_node.rs +++ b/my-no-sql-core/src/db/db_table/db_table_master_node.rs @@ -125,7 +125,7 @@ mod tests { "RowKey": "test", }"#; - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &now).unwrap(); + let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); let db_row = Arc::new(db_row); @@ -149,7 +149,7 @@ mod tests { "RowKey": "test", }"#; - let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &now).unwrap(); + let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); let db_row = Arc::new(db_row); @@ -161,7 +161,7 @@ mod tests { "AAA": "111" }"#; - let db_row2 = DbJsonEntity::parse_into_db_row(test_json.as_bytes(), &now).unwrap(); + let db_row2 = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); let db_row2 = Arc::new(db_row2); From 19badf48ee076a03fcd3e6379f9e806abb032242 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 20 Sep 2024 21:33:00 +0200 Subject: [PATCH 17/89] Added tests --- .github/workflows/build_tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_tests.yaml b/.github/workflows/build_tests.yaml index 10fb9bf..946499e 100644 --- a/.github/workflows/build_tests.yaml +++ b/.github/workflows/build_tests.yaml @@ -19,5 +19,11 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 + - name: Test default + run: cargo test + + - name: Build Master-Node + run: cargo test --master-node + - name: Build default run: cargo build --all-features \ No newline at end of file From ecef0221828dbc0d3e046b26cd860544c3e182af Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 21 Sep 2024 09:17:55 +0200 Subject: [PATCH 18/89] Fixes master node tests --- .../src/db/db_partition/db_rows_container.rs | 20 +++++++++---------- .../src/db/db_row/test_expires_update.rs | 2 +- .../src/db/db_table/db_table_master_node.rs | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/my-no-sql-core/src/db/db_partition/db_rows_container.rs b/my-no-sql-core/src/db/db_partition/db_rows_container.rs index 9830433..95f7170 100644 --- a/my-no-sql-core/src/db/db_partition/db_rows_container.rs +++ b/my-no-sql-core/src/db/db_partition/db_rows_container.rs @@ -165,7 +165,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", "RowKey": "test", - "Expires": "2019-01-01T00:00:00", + "Expires": "2019-01-01T00:00:00" }"#; let time_stamp = JsonTimeStamp::now(); @@ -183,7 +183,7 @@ mod tests { fn test_that_index_does_not_appear_since_we_do_not_have_expiration() { let test_json = r#"{ "PartitionKey": "test", - "RowKey": "test", + "RowKey": "test" }"#; let time_stamp = JsonTimeStamp::now(); @@ -253,7 +253,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", "RowKey": "test", - "Expires": "2019-01-01T00:00:00", + "Expires": "2019-01-01T00:00:00" }"#; let time_stamp = JsonTimeStamp::now(); @@ -292,7 +292,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", "RowKey": "test", - "Expires": "2019-01-01T00:00:00", + "Expires": "2019-01-01T00:00:00" }"#; let now = JsonTimeStamp::now(); @@ -321,7 +321,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", "RowKey": "test", - "Expires": "2019-01-01T00:00:00", + "Expires": "2019-01-01T00:00:00" }"#; let now = JsonTimeStamp::now(); @@ -347,7 +347,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", "RowKey": "test", - "Expires": "2019-01-01T00:00:00", + "Expires": "2019-01-01T00:00:00" }"#; let now = JsonTimeStamp::now(); @@ -372,7 +372,7 @@ mod tests { let json = r#"{ "PartitionKey": "test", - "RowKey": "test1", + "RowKey": "test1" }"#; let db_row = DbJsonEntity::parse_into_db_row( @@ -389,7 +389,7 @@ mod tests { let raw_json = r#"{ "PartitionKey": "test", - "RowKey": "test2", + "RowKey": "test2" }"#; let db_row = DbJsonEntity::parse_into_db_row( @@ -406,7 +406,7 @@ mod tests { let json_db_row = r#"{ "PartitionKey": "test", - "RowKey": "test3", + "RowKey": "test3" }"#; let db_row = DbJsonEntity::parse_into_db_row( @@ -423,7 +423,7 @@ mod tests { let raw_json = r#"{ "PartitionKey": "test", - "RowKey": "test4", + "RowKey": "test4" }"#; let db_row = DbJsonEntity::parse_into_db_row( diff --git a/my-no-sql-core/src/db/db_row/test_expires_update.rs b/my-no-sql-core/src/db/db_row/test_expires_update.rs index 1e96589..5835833 100644 --- a/my-no-sql-core/src/db/db_row/test_expires_update.rs +++ b/my-no-sql-core/src/db/db_row/test_expires_update.rs @@ -10,7 +10,7 @@ mod test { let test_json = r#"{ "PartitionKey": "TestPk", "RowKey": "TestRk", - "Expires": "2019-01-01T00:00:00", + "Expires": "2019-01-01T00:00:00" }"#; let inject_time_stamp = JsonTimeStamp::now(); diff --git a/my-no-sql-core/src/db/db_table/db_table_master_node.rs b/my-no-sql-core/src/db/db_table/db_table_master_node.rs index 0912393..0633795 100644 --- a/my-no-sql-core/src/db/db_table/db_table_master_node.rs +++ b/my-no-sql-core/src/db/db_table/db_table_master_node.rs @@ -122,7 +122,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", - "RowKey": "test", + "RowKey": "test" }"#; let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); @@ -146,7 +146,7 @@ mod tests { let test_json = r#"{ "PartitionKey": "test", - "RowKey": "test", + "RowKey": "test" }"#; let db_row = DbJsonEntity::parse_into_db_row(test_json.as_bytes().into(), &now).unwrap(); From 528cac76200244d447ba72a5d748924b17c51d82 Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 21 Sep 2024 09:19:44 +0200 Subject: [PATCH 19/89] CI Fixes --- .github/workflows/build_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_tests.yaml b/.github/workflows/build_tests.yaml index 946499e..b3c1016 100644 --- a/.github/workflows/build_tests.yaml +++ b/.github/workflows/build_tests.yaml @@ -23,7 +23,7 @@ jobs: run: cargo test - name: Build Master-Node - run: cargo test --master-node + run: cargo test --features master-node - name: Build default run: cargo build --all-features \ No newline at end of file From a53f9f76126bc456dbd3c1dbab3c88df25bf28f9 Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 21 Sep 2024 09:32:04 +0200 Subject: [PATCH 20/89] Fixes --- .../get_entities_builder_mock.rs | 10 +++++++--- .../get_entity_builder/get_entity_builder_mock.rs | 9 ++++++--- .../src/subscribers/my_no_sql_data_reader_mock.rs | 10 ++++++---- .../my_no_sql_data_reader_mock_inner.rs | 14 +++++++++----- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_mock.rs b/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_mock.rs index 324fe77..10b6b2c 100644 --- a/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_mock.rs +++ b/my-no-sql-tcp-reader/src/subscribers/get_entities_builder/get_entities_builder_mock.rs @@ -1,18 +1,22 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::UpdateEntityStatisticsData; use rust_extensions::date_time::DateTimeAsMicroseconds; use crate::subscribers::MyNoSqlDataReaderMockInner; -pub struct GetEntitiesBuilderMock { +pub struct GetEntitiesBuilderMock< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { partition_key: String, update_statistic_data: UpdateEntityStatisticsData, inner: Arc>, } -impl GetEntitiesBuilderMock { +impl + GetEntitiesBuilderMock +{ pub fn new( partition_key: String, inner: Arc>, diff --git a/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_mock.rs b/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_mock.rs index 3037859..cab55b3 100644 --- a/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_mock.rs +++ b/my-no-sql-tcp-reader/src/subscribers/get_entity_builder/get_entity_builder_mock.rs @@ -1,19 +1,22 @@ use std::sync::Arc; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::UpdateEntityStatisticsData; use rust_extensions::date_time::DateTimeAsMicroseconds; use crate::subscribers::MyNoSqlDataReaderMockInner; -pub struct GetEntityBuilderMock<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> { +pub struct GetEntityBuilderMock< + 's, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { partition_key: &'s str, row_key: &'s str, update_statistic_data: UpdateEntityStatisticsData, inner: Arc>, } -impl<'s, TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static> +impl<'s, TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static> GetEntityBuilderMock<'s, TMyNoSqlEntity> { pub fn new( diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs index f012fae..fbff91f 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs @@ -1,18 +1,20 @@ use std::{collections::BTreeMap, sync::Arc}; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use crate::MyNoSqlDataReaderCallBacks; use super::{GetEntitiesBuilder, GetEntityBuilder, MyNoSqlDataReader, MyNoSqlDataReaderMockInner}; -pub struct MyNoSqlDataReaderMock { +pub struct MyNoSqlDataReaderMock< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { pub inner: Arc>, } impl MyNoSqlDataReaderMock where - TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, { pub fn new() -> Self { Self { @@ -31,7 +33,7 @@ where #[async_trait::async_trait] impl MyNoSqlDataReader for MyNoSqlDataReaderMock where - TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, { async fn get_table_snapshot_as_vec(&self) -> Option>> { let result = self.inner.get_table_snapshot_as_vec().await; diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs index c806717..1f87028 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs @@ -3,7 +3,7 @@ use std::{ sync::Arc, }; -use my_no_sql_abstractions::MyNoSqlEntity; +use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::{lazy::LazyVec, AppStates}; use tokio::sync::RwLock; @@ -11,12 +11,14 @@ use crate::MyNoSqlDataReaderCallBacks; use super::MyNoSqlDataReaderCallBacksPusher; -pub struct MyNoSqlDataReaderMockInnerData { +pub struct MyNoSqlDataReaderMockInnerData< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { pub items: BTreeMap>>, pub callbacks: Option>>, } -impl +impl MyNoSqlDataReaderMockInnerData { pub fn new() -> Self { @@ -27,14 +29,16 @@ impl } } -pub struct MyNoSqlDataReaderMockInner { +pub struct MyNoSqlDataReaderMockInner< + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, +> { pub inner: RwLock>, app_states: Arc, } impl MyNoSqlDataReaderMockInner where - TMyNoSqlEntity: MyNoSqlEntity + Sync + Send + 'static, + TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + 'static, { pub fn new() -> Self { Self { From 777d64d5f22f6f629f1ba8b54340fe0c36c8053d Mon Sep 17 00:00:00 2001 From: RomanSavh Date: Tue, 1 Oct 2024 20:28:51 +0300 Subject: [PATCH 21/89] import --- my-no-sql-tcp-reader/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/my-no-sql-tcp-reader/src/lib.rs b/my-no-sql-tcp-reader/src/lib.rs index 60f97fc..e781e1a 100644 --- a/my-no-sql-tcp-reader/src/lib.rs +++ b/my-no-sql-tcp-reader/src/lib.rs @@ -8,7 +8,8 @@ pub use data_reader_entities_set::*; pub use my_no_sql_tcp_connection::MyNoSqlTcpConnection; pub use settings::*; pub use subscribers::{ - MyNoSqlDataReader, MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderData, MyNoSqlDataReaderTcp, + LazyMyNoSqlEntity, MyNoSqlDataReader, MyNoSqlDataReaderCallBacks, MyNoSqlDataReaderData, + MyNoSqlDataReaderTcp, }; #[cfg(feature = "mocks")] From 97a523a1d0cac6f20420a5e3e627d8ab1dadac11 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 3 Oct 2024 12:07:37 +0300 Subject: [PATCH 22/89] Added ability to get list of Partitions --- .../src/my_no_sql_data_writer/execution.rs | 41 +++++++++++++++++++ .../src/my_no_sql_data_writer/fl_url_ext.rs | 19 +++++++++ .../my_no_sql_data_writer.rs | 9 ++++ .../src/my_no_sql_data_writer/with_retries.rs | 9 ++++ .../src/data_reader_entities_set.rs | 7 ++++ .../src/subscribers/my_no_sql_data_reader.rs | 2 + .../subscribers/my_no_sql_data_reader_data.rs | 4 ++ .../subscribers/my_no_sql_data_reader_tcp.rs | 8 ++++ 8 files changed, 99 insertions(+) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 803dede..dde318e 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -11,9 +11,12 @@ use crate::{CreateTableParams, DataWriterError, OperationFailHttpContract, Updat use super::fl_url_ext::FlUrlExt; +const API_SEGMENT: &str = "api"; + const ROW_CONTROLLER: &str = "Row"; const ROWS_CONTROLLER: &str = "Rows"; const BULK_CONTROLLER: &str = "Bulk"; +const PARTITIONS_CONTROLLER: &str = "Partitions"; pub async fn create_table_if_not_exists( flurl: FlUrl, @@ -253,6 +256,7 @@ pub async fn get_by_row_key Result>, DataWriterError> { let mut response = flurl + .append_path_segment(API_SEGMENT) .append_path_segment(ROW_CONTROLLER) .with_row_key_as_query_param(row_key) .with_table_name_as_query_param(TEntity::TABLE_NAME) @@ -273,6 +277,42 @@ pub async fn get_by_row_key, + limit: Option, +) -> Result, DataWriterError> { + let mut response = flurl + .append_path_segment(API_SEGMENT) + .append_path_segment(PARTITIONS_CONTROLLER) + .with_table_name_as_query_param(table_name) + .with_skip_as_query_param(skip) + .with_limit_as_query_param(limit) + .get() + .await?; + + if response.get_status_code() == 404 { + return Err(DataWriterError::TableNotFound(table_name.to_string())); + } + + check_error(&mut response).await?; + + if is_ok_result(&response) { + match serde_json::from_slice(response.get_body_as_slice().await?) { + Ok(entities) => return Ok(entities), + Err(err) => { + return Err(DataWriterError::Error(format!( + "Failed to deserialize: {:?}", + err + ))) + } + } + } + + return Ok(vec![]); +} + pub async fn delete_enum_case< TEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send, TResult: MyNoSqlEntity @@ -319,6 +359,7 @@ pub async fn delete_row Result, DataWriterError> { let mut response = flurl + .append_path_segment(API_SEGMENT) .append_path_segment(ROW_CONTROLLER) .with_partition_key_as_query_param(partition_key) .with_row_key_as_query_param(row_key) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs index cd10959..d8528f4 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_ext.rs @@ -7,6 +7,9 @@ pub trait FlUrlExt { fn with_partition_key_as_query_param(self, partition_key: &str) -> FlUrl; fn with_partition_keys_as_query_param(self, partition_keys: &[&str]) -> FlUrl; fn with_row_key_as_query_param(self, partition_key: &str) -> FlUrl; + + fn with_skip_as_query_param(self, skip: Option) -> FlUrl; + fn with_limit_as_query_param(self, limit: Option) -> FlUrl; } impl FlUrlExt for FlUrl { @@ -43,4 +46,20 @@ impl FlUrlExt for FlUrl { fn with_row_key_as_query_param(self, row_key: &str) -> FlUrl { self.append_query_param("rowKey", Some(row_key)) } + + fn with_skip_as_query_param(self, skip: Option) -> FlUrl { + if let Some(skip) = skip { + self.append_query_param("skip", Some(skip.to_string())) + } else { + self + } + } + + fn with_limit_as_query_param(self, limit: Option) -> FlUrl { + if let Some(limit) = limit { + self.append_query_param("limit", Some(limit.to_string())) + } else { + self + } + } } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index d52a731..98ae5a4 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -218,6 +218,15 @@ impl MyNoSqlData super::execution::get_by_row_key(fl_url, row_key).await } + pub async fn get_partition_keys( + &self, + skip: Option, + limit: Option, + ) -> Result, DataWriterError> { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::get_partition_keys(fl_url, TEntity::TABLE_NAME, skip, limit).await + } + pub async fn delete_enum_case< TResult: MyNoSqlEntity + From diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs index 02db908..97c8874 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs @@ -207,4 +207,13 @@ impl ) .await } + + pub async fn get_partition_keys( + &self, + skip: Option, + limit: Option, + ) -> Result, DataWriterError> { + let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; + super::execution::get_partition_keys(fl_url, TEntity::TABLE_NAME, skip, limit).await + } } diff --git a/my-no-sql-tcp-reader/src/data_reader_entities_set.rs b/my-no-sql-tcp-reader/src/data_reader_entities_set.rs index 88e453d..e3c880f 100644 --- a/my-no-sql-tcp-reader/src/data_reader_entities_set.rs +++ b/my-no-sql-tcp-reader/src/data_reader_entities_set.rs @@ -187,6 +187,13 @@ impl Vec { + match self.entities.as_ref() { + Some(entities) => entities.keys().cloned().collect(), + None => Vec::new(), + } + } } pub struct InitTableResult< diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs index d38c3bf..9876048 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs @@ -23,6 +23,8 @@ pub trait MyNoSqlDataReader< partition_key: &str, ) -> Option>>; + async fn get_partition_keys(&self) -> Vec; + async fn get_entity(&self, partition_key: &str, row_key: &str) -> Option>; async fn get_enum_case_model< diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs index cb23ff5..92d6b1b 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs @@ -89,6 +89,10 @@ where self.entities.delete_rows(rows_to_delete, &self.callbacks); } + pub fn get_partition_keys(&self) -> Vec { + self.entities.get_partition_keys() + } + pub fn get_table_snapshot( &mut self, ) -> Option>>> { diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 82c13a5..829ac25 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -237,6 +237,11 @@ where Some(TResult::from(entity)) } + + pub async fn get_partition_keys(&self) -> Vec { + let write_access = self.inner.data.lock().await; + write_access.get_partition_keys() + } } #[async_trait] @@ -276,6 +281,9 @@ where TMyNoSqlEntity: MyNoSqlEntity + MyNoSqlEntitySerializer + Sync + Send + DeserializeOwned + 'static, { + async fn get_partition_keys(&self) -> Vec { + self.get_partition_keys().await + } async fn get_table_snapshot_as_vec(&self) -> Option>> { self.get_table_snapshot_as_vec().await } From 221fd41f273bbe254dc2e4496bff1642ed6a76c0 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 3 Oct 2024 12:31:30 +0300 Subject: [PATCH 23/89] Get PartitionKeys --- .../src/subscribers/my_no_sql_data_reader_mock.rs | 4 ++++ .../src/subscribers/my_no_sql_data_reader_mock_inner.rs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs index fbff91f..7c5182c 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs @@ -52,6 +52,10 @@ where self.inner.get_by_partition_key(partition_key).await } + async fn get_partition_keys(&self) -> Vec { + self.inner.get_partition_keys().await + } + async fn get_by_partition_key_as_vec( &self, partition_key: &str, diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs index 1f87028..ec62d29 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock_inner.rs @@ -114,6 +114,11 @@ where read_access.items.get(partition_key).cloned() } + pub async fn get_partition_keys(&self) -> Vec { + let read_access = self.inner.read().await; + read_access.items.keys().cloned().collect() + } + pub async fn get_by_partition_key_as_vec( &self, partition_key: &str, From dad249e45b4e52f4b9e9d4e5c59e5535befbdf04 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 3 Oct 2024 12:50:54 +0300 Subject: [PATCH 24/89] Fixes --- .../src/my_no_sql_data_writer/execution.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index dde318e..4400db3 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -6,6 +6,7 @@ use my_json::{ use my_logger::LogEventCtx; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; use rust_extensions::array_of_bytes_iterator::SliceIterator; +use serde::{Deserialize, Serialize}; use crate::{CreateTableParams, DataWriterError, OperationFailHttpContract, UpdateReadStatistics}; @@ -283,6 +284,11 @@ pub async fn get_partition_keys( skip: Option, limit: Option, ) -> Result, DataWriterError> { + #[derive(Serialize, Deserialize)] + pub struct GetPartitionsJsonResult { + pub amount: usize, + pub data: Vec, + } let mut response = flurl .append_path_segment(API_SEGMENT) .append_path_segment(PARTITIONS_CONTROLLER) @@ -299,8 +305,10 @@ pub async fn get_partition_keys( check_error(&mut response).await?; if is_ok_result(&response) { - match serde_json::from_slice(response.get_body_as_slice().await?) { - Ok(entities) => return Ok(entities), + let result: Result = + serde_json::from_slice(response.get_body_as_slice().await?); + match result { + Ok(result) => return Ok(result.data), Err(err) => { return Err(DataWriterError::Error(format!( "Failed to deserialize: {:?}", From 60140bf705ffd2f3932986d31338dd9c5af4a065 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 18 Oct 2024 09:55:33 +0300 Subject: [PATCH 25/89] my-json is exposed --- my-no-sql-core/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/my-no-sql-core/src/lib.rs b/my-no-sql-core/src/lib.rs index 8be0a20..c5aa256 100644 --- a/my-no-sql-core/src/lib.rs +++ b/my-no-sql-core/src/lib.rs @@ -5,4 +5,5 @@ mod expiration_index; pub mod validations; pub use expiration_index::*; pub mod entity_serializer; +pub extern crate my_json; pub extern crate rust_extensions; From 90216df0f44fcc94262977f3d9de13aafc8e81f6 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 18 Oct 2024 11:01:06 +0300 Subject: [PATCH 26/89] zip last version is on --- my-no-sql-tcp-shared/Cargo.toml | 2 +- my-no-sql-tcp-shared/src/payload_compressor.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 5d92866..9a51fda 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -12,4 +12,4 @@ rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-ext tokio = { version = "*", features = ["full"] } tokio-util = "*" async-trait = "*" -zip = { version = "0", default-features = false, features = ["deflate"] } +zip = { version = "*", default-features = false, features = ["deflate"] } diff --git a/my-no-sql-tcp-shared/src/payload_compressor.rs b/my-no-sql-tcp-shared/src/payload_compressor.rs index fef72c6..06f233e 100644 --- a/my-no-sql-tcp-shared/src/payload_compressor.rs +++ b/my-no-sql-tcp-shared/src/payload_compressor.rs @@ -7,8 +7,8 @@ pub fn compress(payload: &[u8]) -> Result, zip::result::ZipError> { { let mut zip = zip::ZipWriter::new(&mut writer); - let options = - zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); zip.start_file("d", options)?; From a34697e2dc6c1ac2791f21977192b0855f91029c Mon Sep 17 00:00:00 2001 From: amigin Date: Sun, 20 Oct 2024 23:35:36 +0300 Subject: [PATCH 27/89] Added ability to scan and find entity inside partition not copying all elements to vec --- .../entity_with_lazy_deserialization.rs | 10 ++++++--- .../subscribers/my_no_sql_data_reader_data.rs | 21 +++++++++++++------ .../subscribers/my_no_sql_data_reader_tcp.rs | 20 ++++++++++++++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs b/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs index bae980d..6843762 100644 --- a/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs +++ b/my-no-sql-tcp-reader/src/subscribers/entity_with_lazy_deserialization.rs @@ -32,16 +32,20 @@ impl Arc { + pub fn get(&mut self) -> &Arc { match self { - LazyMyNoSqlEntity::Deserialized(entity) => entity.clone(), + LazyMyNoSqlEntity::Deserialized(entity) => return entity, LazyMyNoSqlEntity::Raw(src) => { let entity = TMyNoSqlEntity::deserialize_entity(&src.data).unwrap(); let entity = Arc::new(entity); *self = LazyMyNoSqlEntity::Deserialized(entity.clone()); - entity } } + + match self { + LazyMyNoSqlEntity::Deserialized(entity) => entity, + LazyMyNoSqlEntity::Raw(_) => panic!("We should have deserialized it"), + } } pub fn clone(&self) -> Self { diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs index 92d6b1b..ad2b2b1 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_data.rs @@ -101,12 +101,12 @@ where return None; } - let mut result = BTreeMap::new(); + let mut result: BTreeMap>> = BTreeMap::new(); for (partition_key, entities) in entities.iter_mut() { let mut to_insert = BTreeMap::new(); for (row_key, entity) in entities.iter_mut() { - to_insert.insert(row_key.clone(), entity.get()); + to_insert.insert(row_key.clone(), entity.get().clone()); } result.insert(partition_key.clone(), to_insert); @@ -126,7 +126,7 @@ where for partition in entities.values_mut() { for entity in partition.values_mut() { - result.push(entity.get()); + result.push(entity.get().clone()); } } @@ -144,7 +144,7 @@ where let row = partition.get_mut(row_key)?; - Some(row.get()) + Some(row.get().clone()) } pub fn get_by_partition( @@ -158,7 +158,7 @@ where let mut result = BTreeMap::new(); for itm in partition.iter_mut() { - result.insert(itm.0.clone(), itm.1.get()); + result.insert(itm.0.clone(), itm.1.get().clone()); } Some(result) @@ -185,6 +185,15 @@ where Some(result) } + pub fn iter_entities<'s>( + &'s mut self, + partition_key: &str, + ) -> Option>> { + let entities = self.entities.as_mut()?; + let partition = entities.get_mut(partition_key)?; + Some(partition.values_mut()) + } + pub fn has_partition(&self, partition_key: &str) -> bool { let entities = self.entities.as_ref(); @@ -212,7 +221,7 @@ where let mut result = Vec::with_capacity(partition.len()); for db_row in partition.values_mut() { - result.push(db_row.get()); + result.push(db_row.get().clone()); } Some(result) diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 829ac25..361537e 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -113,6 +113,26 @@ where reader.has_partition(partition_key) } + pub async fn iter_and_find_entity_inside_partition( + &self, + partition_key: &str, + predicate: impl Fn(&TMyNoSqlEntity) -> bool, + ) -> Option> { + let mut reader = self.inner.data.lock().await; + + if let Some(entities) = reader.iter_entities(partition_key) { + for entity in entities { + let entity = entity.get(); + + if predicate(&entity) { + return Some(entity.clone()); + } + } + } + + None + } + pub fn deserialize_array( &self, data: &[u8], From de48eec1de369f4e226d9baa84aabd29bdd29dac Mon Sep 17 00:00:00 2001 From: amigin Date: Mon, 21 Oct 2024 18:26:23 +0300 Subject: [PATCH 28/89] Default is implemented for DbTableAttributes --- my-no-sql-core/src/db/db_table/db_table_attributes.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/my-no-sql-core/src/db/db_table/db_table_attributes.rs b/my-no-sql-core/src/db/db_table/db_table_attributes.rs index d03993b..6e042a6 100644 --- a/my-no-sql-core/src/db/db_table/db_table_attributes.rs +++ b/my-no-sql-core/src/db/db_table/db_table_attributes.rs @@ -19,6 +19,12 @@ impl DbTableAttributes { } } +impl Default for DbTableAttributes { + fn default() -> Self { + Self::create_default() + } +} + impl DbTableAttributes { pub fn new( persist: bool, From 4208517ac93be88b8b7937cb1e137fed3f23ccbf Mon Sep 17 00:00:00 2001 From: amigin Date: Mon, 21 Oct 2024 20:49:36 +0300 Subject: [PATCH 29/89] Now we write error if can not find the start of Array or Json Object while parsing json --- my-no-sql-core/Cargo.toml | 2 +- my-no-sql-core/src/db_json_entity/db_json_entity.rs | 8 ++++---- my-no-sql-data-writer/Cargo.toml | 2 +- .../src/my_no_sql_data_writer/execution.rs | 12 +++++++++++- my-no-sql-tcp-reader/Cargo.toml | 2 +- .../src/subscribers/my_no_sql_data_reader_tcp.rs | 11 ++++++++++- 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index 40214bb..c53b722 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -13,7 +13,7 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -my-json = { tag = "0.2.4", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } serde_json = { version = "*" } diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity.rs b/my-no-sql-core/src/db_json_entity/db_json_entity.rs index 0a383c2..0d4c133 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity.rs @@ -246,7 +246,7 @@ impl DbJsonEntity { let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -264,7 +264,7 @@ impl DbJsonEntity { let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -284,7 +284,7 @@ impl DbJsonEntity { let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -316,7 +316,7 @@ impl DbJsonEntity { let mut result = Vec::new(); let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 10eb1eb..836af97 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -19,7 +19,7 @@ rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-ext ] } flurl = { tag = "0.5.3", git = "https://github.com/MyJetTools/fl-url" } -my-json = { tag = "0.2.4", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 4400db3..7484425 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -550,7 +550,17 @@ fn deserialize_entities( ) -> Result, DataWriterError> { let mut result = Vec::new(); let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + let json_array_iterator = JsonArrayIterator::new(slice_iterator); + + if let Err(err) = &json_array_iterator { + panic!( + "Can not deserialize entities for table: {}. Err: {:?}", + TEntity::TABLE_NAME, + err + ); + } + + let mut json_array_iterator = json_array_iterator.unwrap(); while let Some(item) = json_array_iterator.get_next() { let itm = item.unwrap(); diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index 2c52605..e2d4b36 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -16,7 +16,7 @@ my-no-sql-core = { path = "../my-no-sql-core" } rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } my-tcp-sockets = { tag = "0.1.9", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" } -my-json = { tag = "0.2.4", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 361537e..1464af6 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -139,8 +139,17 @@ where ) -> BTreeMap>> { let slice_iterator = SliceIterator::new(data); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator); + let json_array_iterator = JsonArrayIterator::new(slice_iterator); + + if let Err(err) = &json_array_iterator { + panic!( + "Table: {}. The whole array of json entities is broken. Err: {:?}", + TMyNoSqlEntity::TABLE_NAME, + err + ); + } + let mut json_array_iterator = json_array_iterator.unwrap(); let mut result = BTreeMap::new(); while let Some(db_entity) = json_array_iterator.get_next() { From f0118116abefe152086232821e8e742af374e382 Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 9 Nov 2024 21:51:33 +0100 Subject: [PATCH 30/89] plugged my-http-client --- my-no-sql-abstractions/Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 2 +- my-no-sql-data-writer/Cargo.toml | 4 ++-- my-no-sql-macros/Cargo.toml | 2 +- my-no-sql-sdk/Cargo.toml | 2 +- my-no-sql-tcp-reader/Cargo.toml | 2 +- my-no-sql-tcp-shared/Cargo.toml | 2 +- my-no-sql-tests/Cargo.toml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/my-no-sql-abstractions/Cargo.toml b/my-no-sql-abstractions/Cargo.toml index 240b254..d728921 100644 --- a/my-no-sql-abstractions/Cargo.toml +++ b/my-no-sql-abstractions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-abstractions" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index c53b722..f91d579 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-core" -version = "0.3.3" +version = "0.4.0" edition = "2021" [features] diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 836af97..9fdf221 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-data-writer" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -17,7 +17,7 @@ my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ "with-tokio", ] } -flurl = { tag = "0.5.3", git = "https://github.com/MyJetTools/fl-url" } +flurl = { tag = "0.6.0", git = "https://github.com/MyJetTools/fl-url" } my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } diff --git a/my-no-sql-macros/Cargo.toml b/my-no-sql-macros/Cargo.toml index a453887..60f0210 100644 --- a/my-no-sql-macros/Cargo.toml +++ b/my-no-sql-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-macros" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index 34c3e81..2ff1344 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-sdk" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index e2d4b36..6fddac2 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-reader" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 9a51fda..5d149fa 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-shared" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-tests/Cargo.toml b/my-no-sql-tests/Cargo.toml index 01e3770..c77b284 100644 --- a/my-no-sql-tests/Cargo.toml +++ b/my-no-sql-tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tests" -version = "0.3.3" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From acfa1561c5cf3721da9c9a0302be5626628e2da2 Mon Sep 17 00:00:00 2001 From: amigin Date: Mon, 11 Nov 2024 11:40:11 +0100 Subject: [PATCH 31/89] New api to set ssh credentials --- my-no-sql-data-writer/src/lib.rs | 3 +++ .../my_no_sql_data_writer/fl_url_factory.rs | 11 +++++------ .../my_no_sql_data_writer.rs | 19 +++++++++++-------- my-no-sql-data-writer/src/ssh.rs | 8 ++++++++ 4 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 my-no-sql-data-writer/src/ssh.rs diff --git a/my-no-sql-data-writer/src/lib.rs b/my-no-sql-data-writer/src/lib.rs index 4cdabcb..244cbe0 100644 --- a/my-no-sql-data-writer/src/lib.rs +++ b/my-no-sql-data-writer/src/lib.rs @@ -1,2 +1,5 @@ mod my_no_sql_data_writer; pub use my_no_sql_data_writer::*; + +#[cfg(feature = "with-ssh")] +mod ssh; diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index a0bc886..bfb2add 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -18,8 +18,8 @@ pub struct FlUrlFactory { #[cfg(feature = "with-ssh")] pub http_buffer_size: Option, #[cfg(feature = "with-ssh")] - pub ssh_cert_credentials: - Option>, + pub ssh_credentials: Arc, + create_table_is_called: Arc>, table_name: &'static str, } @@ -37,10 +37,10 @@ impl FlUrlFactory { create_table_is_called: UnsafeValue::new(false).into(), settings, table_name, - #[cfg(feature = "with-ssh")] - ssh_cert_credentials: None, + #[cfg(feature = "with-ssh")] http_buffer_size: None, + ssh_credentials: Arc::new(crate::ssh::SshCredentials::UserAgent), } } #[cfg(not(feature = "with-ssh"))] @@ -50,8 +50,7 @@ impl FlUrlFactory { } #[cfg(feature = "with-ssh")] async fn create_fl_url(&self, url: &str) -> FlUrl { - let mut fl_url = - flurl::FlUrl::new_with_maybe_ssh(url, self.ssh_cert_credentials.as_ref()).await; + let mut fl_url = flurl::FlUrl::new(url); if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 98ae5a4..37a1a1f 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -80,14 +80,17 @@ impl MyNoSqlData } #[cfg(feature = "with-ssh")] - pub fn set_ssh_cert_credentials( - &mut self, - cert_credentials: std::collections::HashMap< - String, - flurl::my_ssh::SshCredentialsSettingsModel, - >, - ) { - self.fl_url_factory.ssh_cert_credentials = Some(cert_credentials); + pub fn set_ssh_password(&mut self, password: String) { + self.fl_url_factory.ssh_credentials = + Arc::new(crate::ssh::SshCredentials::Password(password)); + } + + #[cfg(feature = "with-ssh")] + pub fn set_ssh_private_key(&mut self, private_key: String, passphrase: String) { + self.fl_url_factory.ssh_credentials = Arc::new(crate::ssh::SshCredentials::PrivateKey { + private_key, + passphrase, + }); } #[cfg(feature = "with-ssh")] diff --git a/my-no-sql-data-writer/src/ssh.rs b/my-no-sql-data-writer/src/ssh.rs new file mode 100644 index 0000000..2cc914e --- /dev/null +++ b/my-no-sql-data-writer/src/ssh.rs @@ -0,0 +1,8 @@ +pub enum SshCredentials { + UserAgent, + Password(String), + PrivateKey { + private_key: String, + passphrase: String, + }, +} From 18a3611218d2381baa3d35f8dcf708e1a6597c63 Mon Sep 17 00:00:00 2001 From: amigin Date: Mon, 11 Nov 2024 11:43:08 +0100 Subject: [PATCH 32/89] Fixes --- .../src/my_no_sql_data_writer/fl_url_factory.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index bfb2add..c9a1f10 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -40,6 +40,7 @@ impl FlUrlFactory { #[cfg(feature = "with-ssh")] http_buffer_size: None, + #[cfg(feature = "with-ssh")] ssh_credentials: Arc::new(crate::ssh::SshCredentials::UserAgent), } } From 408d021267c260033db4002040ed7697052e2b50 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 13 Nov 2024 12:03:27 +0100 Subject: [PATCH 33/89] Added ability to ping writer --- my-no-sql-data-writer/Cargo.toml | 1 + my-no-sql-data-writer/src/lib.rs | 7 + .../src/my_no_sql_data_writer/mod.rs | 1 + .../my_no_sql_data_writer.rs | 8 +- my-no-sql-data-writer/src/ping_pool.rs | 141 ++++++++++++++++++ 5 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 my-no-sql-data-writer/src/ping_pool.rs diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 9fdf221..7ced911 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -27,6 +27,7 @@ async-trait = "*" serde = { version = "*", features = ["derive"] } serde_json = "*" serde_derive = "*" +lazy_static = "*" [dev-dependencies] my-no-sql-core = { path = "../my-no-sql-core" } diff --git a/my-no-sql-data-writer/src/lib.rs b/my-no-sql-data-writer/src/lib.rs index 244cbe0..d1ad64e 100644 --- a/my-no-sql-data-writer/src/lib.rs +++ b/my-no-sql-data-writer/src/lib.rs @@ -3,3 +3,10 @@ pub use my_no_sql_data_writer::*; #[cfg(feature = "with-ssh")] mod ssh; + +mod ping_pool; +pub use ping_pool::*; + +lazy_static::lazy_static! { + static ref PING_POOL: crate::PingPool = crate::PingPool::new(); +} diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs index 76d6049..fcee8b0 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/mod.rs @@ -11,3 +11,4 @@ mod fl_url_ext; mod with_retries; pub use with_retries::*; mod fl_url_factory; +pub use fl_url_factory::*; diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 37a1a1f..615bd5e 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -50,11 +50,17 @@ pub struct MyNoSqlDataWriter { } impl MyNoSqlDataWriter { - pub fn new( + pub async fn new( + app_name: &str, + app_version: &str, settings: Arc, auto_create_table_params: Option, sync_period: DataSynchronizationPeriod, ) -> Self { + crate::PING_POOL + .register(app_name, app_version, settings.clone(), TEntity::TABLE_NAME) + .await; + Self { phantom: PhantomData, sync_period, diff --git a/my-no-sql-data-writer/src/ping_pool.rs b/my-no-sql-data-writer/src/ping_pool.rs new file mode 100644 index 0000000..5a166ab --- /dev/null +++ b/my-no-sql-data-writer/src/ping_pool.rs @@ -0,0 +1,141 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; + +use crate::{FlUrlFactory, MyNoSqlWriterSettings}; + +pub struct PingDataItem { + pub name: String, + pub version: String, + + pub table_settings: Vec<( + String, + Arc, + )>, +} + +pub struct PingPoolInner { + items: Vec, + started: bool, +} + +impl PingPoolInner { + pub fn new() -> Self { + Self { + items: Vec::new(), + started: false, + } + } +} + +pub struct PingPool { + data: Mutex, +} + +impl PingPool { + pub fn new() -> Self { + Self { + data: Mutex::new(PingPoolInner::new()), + } + } + + pub async fn register( + &self, + name: &str, + version: &str, + settings: Arc, + table: &str, + ) { + let mut data = self.data.lock().await; + if !data.started { + tokio::spawn(async move { ping_loop().await }); + data.started = true; + } + + let index = data + .items + .iter() + .position(|x| x.name == name && x.version == version); + + if let Some(index) = index { + let item = &mut data.items[index]; + item.table_settings.push((table.to_string(), settings)); + } else { + let item = PingDataItem { + name: name.to_string(), + version: version.to_string(), + + table_settings: vec![((table.to_string(), settings))], + }; + + data.items.push(item); + } + } +} + +async fn ping_loop() { + let delay = Duration::from_secs(30); + loop { + tokio::time::sleep(delay).await; + + let access = crate::PING_POOL.data.lock().await; + + for itm in access.items.iter() { + let mut url_to_ping = HashMap::new(); + for (table, settings) in itm.table_settings.iter() { + let url = settings.get_url().await; + let entry = url_to_ping + .entry(url) + .or_insert_with(|| ((settings.clone(), Vec::new()))); + entry.1.push(table.to_string()); + } + + for (_, (settings, tables)) in url_to_ping { + let factory = FlUrlFactory::new(settings, None, ""); + + let ping_model = PingModel { + name: itm.name.to_string(), + version: itm.version.to_string(), + tables, + }; + + let fl_url = factory.get_fl_url().await; + + if let Err(err) = &fl_url { + println!("{}:{} ping error: {:?}", itm.name, itm.version, err); + continue; + } + + let fl_url_response = fl_url + .unwrap() + .0 + .append_path_segment("api") + .append_path_segment("ping") + .post_json(&ping_model) + .await; + + if let Err(err) = &fl_url_response { + println!("{}:{} ping error: {:?}", itm.name, itm.version, err); + continue; + } + + let fl_url_response = fl_url_response.unwrap(); + + println!( + "{}:{} ping status code {}", + itm.name, + itm.version, + fl_url_response.get_status_code() + ) + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PingModel { + pub name: String, + pub version: String, + pub tables: Vec, +} From b6442ac80e1a0db17973cee83308db33f6586ca5 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 13 Nov 2024 12:07:50 +0100 Subject: [PATCH 34/89] Added get_app_name and get_app_version as part of settings --- .../my_no_sql_data_writer.rs | 4 +--- .../src/my_no_sql_data_writer/settings.rs | 2 ++ my-no-sql-data-writer/src/ping_pool.rs | 18 ++++++++---------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 615bd5e..d360331 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -51,14 +51,12 @@ pub struct MyNoSqlDataWriter { impl MyNoSqlDataWriter { pub async fn new( - app_name: &str, - app_version: &str, settings: Arc, auto_create_table_params: Option, sync_period: DataSynchronizationPeriod, ) -> Self { crate::PING_POOL - .register(app_name, app_version, settings.clone(), TEntity::TABLE_NAME) + .register(settings.clone(), TEntity::TABLE_NAME) .await; Self { diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/settings.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/settings.rs index 70d62ef..651b764 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/settings.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/settings.rs @@ -1,4 +1,6 @@ #[async_trait::async_trait] pub trait MyNoSqlWriterSettings { async fn get_url(&self) -> String; + fn get_app_name(&self) -> &'static str; + fn get_app_version(&self) -> &'static str; } diff --git a/my-no-sql-data-writer/src/ping_pool.rs b/my-no-sql-data-writer/src/ping_pool.rs index 5a166ab..4241407 100644 --- a/my-no-sql-data-writer/src/ping_pool.rs +++ b/my-no-sql-data-writer/src/ping_pool.rs @@ -6,8 +6,8 @@ use tokio::sync::Mutex; use crate::{FlUrlFactory, MyNoSqlWriterSettings}; pub struct PingDataItem { - pub name: String, - pub version: String, + pub name: &'static str, + pub version: &'static str, pub table_settings: Vec<( String, @@ -42,8 +42,7 @@ impl PingPool { pub async fn register( &self, - name: &str, - version: &str, + settings: Arc, table: &str, ) { @@ -53,18 +52,17 @@ impl PingPool { data.started = true; } - let index = data - .items - .iter() - .position(|x| x.name == name && x.version == version); + let index = data.items.iter().position(|x| { + x.name == settings.get_app_name() && x.version == settings.get_app_version() + }); if let Some(index) = index { let item = &mut data.items[index]; item.table_settings.push((table.to_string(), settings)); } else { let item = PingDataItem { - name: name.to_string(), - version: version.to_string(), + name: settings.get_app_name(), + version: settings.get_app_version(), table_settings: vec![((table.to_string(), settings))], }; From 77309365127a88ecf490c988e9dedd1b9807bfb1 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 13 Nov 2024 12:52:12 +0100 Subject: [PATCH 35/89] Fixes --- .../src/my_no_sql_data_writer/fl_url_factory.rs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index c9a1f10..1944d20 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -44,25 +44,11 @@ impl FlUrlFactory { ssh_credentials: Arc::new(crate::ssh::SshCredentials::UserAgent), } } - #[cfg(not(feature = "with-ssh"))] + async fn create_fl_url(&self, url: &str) -> FlUrl { let fl_url = flurl::FlUrl::new(url); fl_url } - #[cfg(feature = "with-ssh")] - async fn create_fl_url(&self, url: &str) -> FlUrl { - let mut fl_url = flurl::FlUrl::new(url); - - if let Some(ssh_sessions_pool) = &self.ssh_sessions_pool { - fl_url = fl_url.set_ssh_sessions_pool(ssh_sessions_pool.clone()); - } - - if let Some(http_buffer_size) = self.http_buffer_size { - fl_url = fl_url.set_http_buffer_size(http_buffer_size); - } - - fl_url - } pub async fn get_fl_url(&self) -> Result<(FlUrl, String), DataWriterError> { let url = self.settings.get_url().await; From 657e6033e166253863f582028f1278b1038ae399 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 13 Nov 2024 13:47:55 +0100 Subject: [PATCH 36/89] Return backward compatible --- .../my_no_sql_data_writer/my_no_sql_data_writer.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index d360331..18bcd75 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -50,14 +50,17 @@ pub struct MyNoSqlDataWriter { } impl MyNoSqlDataWriter { - pub async fn new( + pub fn new( settings: Arc, auto_create_table_params: Option, sync_period: DataSynchronizationPeriod, ) -> Self { - crate::PING_POOL - .register(settings.clone(), TEntity::TABLE_NAME) - .await; + let settings_cloned = settings.clone(); + tokio::spawn(async move { + crate::PING_POOL + .register(settings_cloned, TEntity::TABLE_NAME) + .await; + }); Self { phantom: PhantomData, From ad35369ac00dc741678d8e6b953131663ddc14b4 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 19 Nov 2024 19:52:31 +0200 Subject: [PATCH 37/89] rust-extensions:0.1.5 --- my-no-sql-abstractions/Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 6 +++--- my-no-sql-data-writer/Cargo.toml | 10 +++++----- my-no-sql-macros/Cargo.toml | 4 ++-- my-no-sql-sdk/Cargo.toml | 4 ++-- my-no-sql-tcp-reader/Cargo.toml | 10 +++++----- my-no-sql-tcp-reader/src/my_no_sql_tcp_connection.rs | 6 +++++- my-no-sql-tcp-reader/src/tcp_events.rs | 10 +++------- my-no-sql-tcp-shared/Cargo.toml | 6 +++--- my-no-sql-tcp-shared/src/tcp_contracts.rs | 10 +++++++++- my-no-sql-tcp-shared/src/tcp_serializer.rs | 9 +-------- my-no-sql-tests/Cargo.toml | 2 +- 12 files changed, 40 insertions(+), 39 deletions(-) diff --git a/my-no-sql-abstractions/Cargo.toml b/my-no-sql-abstractions/Cargo.toml index d728921..8db7846 100644 --- a/my-no-sql-abstractions/Cargo.toml +++ b/my-no-sql-abstractions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-abstractions" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index f91d579..4e9e9be 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-core" -version = "0.4.0" +version = "0.4.1" edition = "2021" [features] @@ -13,8 +13,8 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } -rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } +my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } +rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } serde_json = { version = "*" } serde = { version = "*", features = ["derive"] } diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 7ced911..8ccb1d4 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-data-writer" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -12,14 +12,14 @@ with-ssh = ["flurl/with-ssh"] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" } +my-logger = { tag = "1.2.1", git = "https://github.com/MyJetTools/my-logger.git" } -rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ +rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ "with-tokio", ] } -flurl = { tag = "0.6.0", git = "https://github.com/MyJetTools/fl-url" } +flurl = { tag = "0.6.1", git = "https://github.com/MyJetTools/fl-url" } -my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-macros/Cargo.toml b/my-no-sql-macros/Cargo.toml index 60f0210..dff5d7b 100644 --- a/my-no-sql-macros/Cargo.toml +++ b/my-no-sql-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-macros" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -9,7 +9,7 @@ edition = "2021" proc-macro = true [dependencies] -types-reader = { tag = "0.5.0", git = "https://github.com/MyJetTools/types-reader.git" } +types-reader = { tag = "0.5.1", git = "https://github.com/MyJetTools/types-reader.git" } syn = { version = "*", features = ["extra-traits"] } diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index 2ff1344..54f2aba 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-sdk" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -25,5 +25,5 @@ my-no-sql-macros = { optional = true, path = "../my-no-sql-macros" } my-no-sql-tcp-reader = { optional = true, path = "../my-no-sql-tcp-reader" } my-no-sql-tcp-shared = { optional = true, path = "../my-no-sql-tcp-shared" } -rust-extensions = { optional = true, tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } +rust-extensions = { optional = true, tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } # my_no_sql::macros::my_no_sql_entity diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index 6fddac2..9af20d3 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tcp-reader" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -13,10 +13,10 @@ my-no-sql-tcp-shared = { path = "../my-no-sql-tcp-shared" } my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } my-no-sql-core = { path = "../my-no-sql-core" } -rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } -my-tcp-sockets = { tag = "0.1.9", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } -my-logger = { tag = "1.1.0", git = "https://github.com/MyJetTools/my-logger.git" } -my-json = { tag = "0.3.0", git = "https://github.com/MyJetTools/my-json.git" } +rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } +my-tcp-sockets = { tag = "0.1.11", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } +my-logger = { tag = "1.2.1", git = "https://github.com/MyJetTools/my-logger.git" } +my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-tcp-reader/src/my_no_sql_tcp_connection.rs b/my-no-sql-tcp-reader/src/my_no_sql_tcp_connection.rs index de05a83..1c73151 100644 --- a/my-no-sql-tcp-reader/src/my_no_sql_tcp_connection.rs +++ b/my-no-sql-tcp-reader/src/my_no_sql_tcp_connection.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::{sync_to_main::SyncToMainNodeHandler, MyNoSqlTcpSerializerFactory}; -use my_tcp_sockets::TcpClient; +use my_tcp_sockets::{TcpClient, TlsSettings}; use rust_extensions::{AppStates, StrOrString}; use crate::{ @@ -18,6 +18,10 @@ impl my_tcp_sockets::TcpClientSocketSettings for TcpConnectionSettings { async fn get_host_port(&self) -> Option { self.settings.get_host_port().await.into() } + + async fn get_tls_settings(&self) -> Option { + None + } } pub struct MyNoSqlTcpConnection { diff --git a/my-no-sql-tcp-reader/src/tcp_events.rs b/my-no-sql-tcp-reader/src/tcp_events.rs index 0b00fe4..a4fec08 100644 --- a/my-no-sql-tcp-reader/src/tcp_events.rs +++ b/my-no-sql-tcp-reader/src/tcp_events.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use my_no_sql_tcp_shared::{ sync_to_main::SyncToMainNodeHandler, MyNoSqlReaderTcpSerializer, MyNoSqlTcpContract, }; -use my_tcp_sockets::{tcp_connection::TcpSocketConnection, SocketEventCallback}; +use my_tcp_sockets::{ + tcp_connection::TcpSocketConnection, SocketEventCallback, TcpSerializerState, +}; use crate::subscribers::Subscribers; @@ -22,12 +24,6 @@ impl TcpEvents { sync_handler, } } - pub async fn handle_incoming_packet( - &self, - _tcp_contract: MyNoSqlTcpContract, - _connection: Arc, - ) { - } } #[async_trait::async_trait] diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 5d149fa..0c4eb28 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "my-no-sql-tcp-shared" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -my-tcp-sockets = { tag = "0.1.9", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } -rust-extensions = { tag = "0.1.4", git = "https://github.com/MyJetTools/rust-extensions.git" } +my-tcp-sockets = { tag = "0.1.11", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } +rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-tcp-shared/src/tcp_contracts.rs b/my-no-sql-tcp-shared/src/tcp_contracts.rs index 59b5b36..f45020e 100644 --- a/my-no-sql-tcp-shared/src/tcp_contracts.rs +++ b/my-no-sql-tcp-shared/src/tcp_contracts.rs @@ -1,6 +1,6 @@ use my_tcp_sockets::{ socket_reader::{ReadingTcpContractFail, SocketReader, SocketReaderInMem}, - TcpWriteBuffer, + TcpSerializerState, TcpWriteBuffer, }; use rust_extensions::date_time::DateTimeAsMicroseconds; @@ -496,3 +496,11 @@ impl my_tcp_sockets::TcpContract for MyNoSqlTcpContract { } } } + +impl TcpSerializerState for () { + fn is_tcp_contract_related_to_metadata(&self, contract: &MyNoSqlTcpContract) -> bool { + false + } + + fn apply_tcp_contract(&mut self, contract: &MyNoSqlTcpContract) {} +} diff --git a/my-no-sql-tcp-shared/src/tcp_serializer.rs b/my-no-sql-tcp-shared/src/tcp_serializer.rs index b93b99d..af861f1 100644 --- a/my-no-sql-tcp-shared/src/tcp_serializer.rs +++ b/my-no-sql-tcp-shared/src/tcp_serializer.rs @@ -1,6 +1,6 @@ use my_tcp_sockets::{ socket_reader::{ReadingTcpContractFail, SocketReader}, - TcpSerializerFactory, TcpSerializerState, TcpSocketSerializer, TcpWriteBuffer, + TcpSerializerFactory, TcpSocketSerializer, TcpWriteBuffer, }; use crate::MyNoSqlTcpContract; @@ -38,13 +38,6 @@ impl TcpSocketSerializer for MyNoSqlReaderTcpSerializer } } -impl TcpSerializerState for () { - fn is_tcp_contract_related_to_metadata(&self, _: &MyNoSqlTcpContract) -> bool { - false - } - fn apply_tcp_contract(&mut self, _: &MyNoSqlTcpContract) {} -} - pub struct MyNoSqlTcpSerializerFactory; #[async_trait::async_trait] diff --git a/my-no-sql-tests/Cargo.toml b/my-no-sql-tests/Cargo.toml index c77b284..9a3129b 100644 --- a/my-no-sql-tests/Cargo.toml +++ b/my-no-sql-tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "my-no-sql-tests" -version = "0.4.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 80420a7be4481b3fe487de15d3fdeaf79d460725 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 19 Nov 2024 21:02:47 +0200 Subject: [PATCH 38/89] Fixes --- .../src/db_json_entity/db_json_entity.rs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity.rs b/my-no-sql-core/src/db_json_entity/db_json_entity.rs index 0d4c133..6e29970 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity.rs @@ -1,6 +1,7 @@ use crate::db::DbRow; use my_json::json_reader::array_iterator::JsonArrayIterator; +use my_json::json_reader::JsonArrayIteratorFromSlice; use rust_extensions::array_of_bytes_iterator::SliceIterator; use rust_extensions::date_time::DateTimeAsMicroseconds; @@ -28,7 +29,7 @@ impl DbJsonEntity { Self::new(JsonFirstLineReader::new(slice_iterator)) } pub fn new( - mut json_first_line_reader: JsonFirstLineReader, + json_first_line_reader: JsonFirstLineReader, ) -> Result { let mut partition_key = None; let mut row_key = None; @@ -40,7 +41,7 @@ impl DbJsonEntity { while let Some(line) = json_first_line_reader.get_next() { let line = line?; - let name = line.name.as_unescaped_name(&json_first_line_reader)?; + let name = line.name.as_unescaped_str(&json_first_line_reader)?; match name { super::consts::PARTITION_KEY => { partition_key = Some(JsonKeyValuePosition::new(&line)); @@ -118,7 +119,7 @@ impl DbJsonEntity { } pub fn parse_into_db_row( - mut json_first_line_reader: JsonFirstLineReader, + json_first_line_reader: JsonFirstLineReader, now: &JsonTimeStamp, ) -> Result { let mut partition_key = None; @@ -134,7 +135,7 @@ impl DbJsonEntity { let line_ref = line.as_ref(&json_first_line_reader); - let name = line_ref.name.as_unescaped_name().unwrap(); + let name = line_ref.name.as_unescaped_str().unwrap(); match name { super::consts::PARTITION_KEY => { partition_key = Some(raw.append(line_ref)); @@ -246,7 +247,7 @@ impl DbJsonEntity { let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -264,7 +265,7 @@ impl DbJsonEntity { let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -284,7 +285,7 @@ impl DbJsonEntity { let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIterator::new(slice_iterator)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -315,14 +316,11 @@ impl DbJsonEntity { ) -> Result>)>, DbEntityParseFail> { let mut result = Vec::new(); - let slice_iterator = SliceIterator::new(src); - let mut json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIteratorFromSlice::new(src)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_row = DbJsonEntity::restore_into_db_row( - json.unwrap_as_object(&json_array_iterator).unwrap(), - )?; + let db_row = DbJsonEntity::restore_into_db_row(json.unwrap_as_object().unwrap())?; let partition_key = db_row.get_partition_key(); From ba3da21dd1044de3603c4f5c20d6d41ea46db4e1 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 20 Nov 2024 00:10:03 +0200 Subject: [PATCH 39/89] Improvements with Parsing DbEntity --- .../src/db_json_entity/db_json_entity.rs | 158 +++++++----------- .../db_json_entity_with_content.rs | 6 +- .../db_row_content_compiler.rs | 13 +- .../db_json_entity/json_key_value_position.rs | 12 +- my-no-sql-core/src/entity_serializer.rs | 7 +- .../src/my_no_sql_data_writer/execution.rs | 11 +- .../subscribers/my_no_sql_data_reader_tcp.rs | 20 +-- my-no-sql-tcp-reader/src/tcp_events.rs | 23 +-- my-no-sql-tcp-shared/src/tcp_contracts.rs | 4 +- 9 files changed, 103 insertions(+), 151 deletions(-) diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity.rs b/my-no-sql-core/src/db_json_entity/db_json_entity.rs index 6e29970..ee7dc91 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity.rs @@ -1,8 +1,7 @@ use crate::db::DbRow; -use my_json::json_reader::array_iterator::JsonArrayIterator; -use my_json::json_reader::JsonArrayIteratorFromSlice; -use rust_extensions::array_of_bytes_iterator::SliceIterator; +use my_json::json_reader::JsonArrayIterator; + use rust_extensions::date_time::DateTimeAsMicroseconds; use std::sync::Arc; @@ -13,7 +12,7 @@ use super::DbRowContentCompiler; use super::JsonKeyValuePosition; use super::JsonTimeStamp; use super::KeyValueContentPosition; -use my_json::json_reader::JsonFirstLineReader; +use my_json::json_reader::JsonFirstLineIterator; pub struct DbJsonEntity { pub partition_key: JsonKeyValuePosition, @@ -25,12 +24,9 @@ pub struct DbJsonEntity { impl DbJsonEntity { pub fn from_slice(src: &[u8]) -> Result { - let slice_iterator = SliceIterator::new(src); - Self::new(JsonFirstLineReader::new(slice_iterator)) + Self::new(JsonFirstLineIterator::new(src)) } - pub fn new( - json_first_line_reader: JsonFirstLineReader, - ) -> Result { + pub fn new(json_first_line_reader: JsonFirstLineIterator) -> Result { let mut partition_key = None; let mut row_key = None; let mut expires = None; @@ -39,63 +35,55 @@ impl DbJsonEntity { let mut expires_value = None; while let Some(line) = json_first_line_reader.get_next() { - let line = line?; + let (name_ref, value_ref) = line?; - let name = line.name.as_unescaped_str(&json_first_line_reader)?; + let name = name_ref.as_unescaped_str()?; match name { super::consts::PARTITION_KEY => { - partition_key = Some(JsonKeyValuePosition::new(&line)); + if value_ref.as_str().is_none() { + return Err(DbEntityParseFail::FieldPartitionKeyCanNotBeNull); + } + + partition_key = + Some(JsonKeyValuePosition::new(&name_ref.data, &value_ref.data)); } super::consts::ROW_KEY => { - row_key = Some(JsonKeyValuePosition::new(&line)); + if value_ref.as_str().is_none() { + return Err(DbEntityParseFail::FieldRowKeyCanNotBeNull); + } + row_key = Some(JsonKeyValuePosition::new(&name_ref.data, &value_ref.data)); } super::consts::EXPIRES => { - expires_value = line.value.as_date_time(&json_first_line_reader); - expires = Some(JsonKeyValuePosition::new(&line)) + expires_value = value_ref.as_date_time(); + expires = Some(JsonKeyValuePosition::new(&name_ref.data, &value_ref.data)); } super::consts::TIME_STAMP => { - time_stamp = Some(JsonKeyValuePosition::new(&line)); + time_stamp = Some(JsonKeyValuePosition::new(&name_ref.data, &value_ref.data)); } _ => { if rust_extensions::str_utils::compare_strings_case_insensitive( name, super::consts::TIME_STAMP_LOWER_CASE, ) { - time_stamp = Some(JsonKeyValuePosition::new(&line)); + time_stamp = + Some(JsonKeyValuePosition::new(&name_ref.data, &value_ref.data)); } } } } - let raw = json_first_line_reader.get_src_slice(); if partition_key.is_none() { return Err(DbEntityParseFail::FieldPartitionKeyIsRequired); } - let partition_key = partition_key.unwrap(); - - if partition_key.key.len() > 255 { - return Err(DbEntityParseFail::PartitionKeyIsTooLong); - } - - if partition_key.value.is_null(raw) { - return Err(DbEntityParseFail::FieldPartitionKeyCanNotBeNull); - } - if row_key.is_none() { return Err(DbEntityParseFail::FieldRowKeyIsRequired); } - let row_key = row_key.unwrap(); - - if row_key.value.is_null(raw) { - return Err(DbEntityParseFail::FieldRowKeyCanNotBeNull); - } - let result = Self { - partition_key, - row_key, + partition_key: partition_key.unwrap(), + row_key: row_key.unwrap(), expires, time_stamp, expires_value, @@ -108,8 +96,7 @@ impl DbJsonEntity { raw: &'s [u8], time_stamp_to_inject: &'s JsonTimeStamp, ) -> Result, DbEntityParseFail> { - let slice_iterator = SliceIterator::new(raw); - let entity = Self::new(JsonFirstLineReader::new(slice_iterator))?; + let entity = Self::new(JsonFirstLineIterator::new(raw))?; return Ok(DbJsonEntityWithContent::new( raw, @@ -119,7 +106,7 @@ impl DbJsonEntity { } pub fn parse_into_db_row( - json_first_line_reader: JsonFirstLineReader, + json_first_line_reader: JsonFirstLineIterator, now: &JsonTimeStamp, ) -> Result { let mut partition_key = None; @@ -128,28 +115,29 @@ impl DbJsonEntity { let mut time_stamp = None; let mut expires_value = None; - let mut raw = DbRowContentCompiler::new(json_first_line_reader.get_src_slice().len()); + let mut raw = DbRowContentCompiler::new(json_first_line_reader.as_slice().len()); while let Some(line) = json_first_line_reader.get_next() { - let line = line?; - - let line_ref = line.as_ref(&json_first_line_reader); + let (name_ref, value_ref) = line?; - let name = line_ref.name.as_unescaped_str().unwrap(); + let name = name_ref.as_unescaped_str().unwrap(); match name { super::consts::PARTITION_KEY => { - partition_key = Some(raw.append(line_ref)); + partition_key = Some(raw.append(&name_ref, &value_ref)); + + println!("PartitionKey: {:?}", partition_key); } super::consts::ROW_KEY => { - row_key = Some(raw.append(line_ref)); + row_key = Some(raw.append(&name_ref, &value_ref)); + println!("RowKey: {:?}", row_key); time_stamp = raw .append_str_value(super::consts::TIME_STAMP, now.as_str()) .into(); } super::consts::EXPIRES => { - expires_value = line.value.as_date_time(&json_first_line_reader); - expires = Some(raw.append(line_ref)); + expires_value = value_ref.as_date_time(); + expires = Some(raw.append(&name_ref, &value_ref)); } super::consts::TIME_STAMP => {} _ => { @@ -158,12 +146,14 @@ impl DbJsonEntity { super::consts::TIME_STAMP_LOWER_CASE, ) { } else { - raw.append(line_ref); + raw.append(&name_ref, &value_ref); } } } } + let content = raw.into_vec(); + if partition_key.is_none() { return Err(DbEntityParseFail::FieldPartitionKeyIsRequired); } @@ -174,10 +164,7 @@ impl DbJsonEntity { return Err(DbEntityParseFail::PartitionKeyIsTooLong); } - if partition_key - .value - .is_null(json_first_line_reader.get_src_slice()) - { + if partition_key.value.is_null(content.as_slice()) { return Err(DbEntityParseFail::FieldPartitionKeyCanNotBeNull); } @@ -187,10 +174,7 @@ impl DbJsonEntity { let row_key = row_key.unwrap(); - if row_key - .value - .is_null(json_first_line_reader.get_src_slice()) - { + if row_key.value.is_null(content.as_slice()) { return Err(DbEntityParseFail::FieldRowKeyCanNotBeNull); } @@ -202,7 +186,7 @@ impl DbJsonEntity { expires_value, }; - let result = DbRow::new(db_json_entity, raw.into_vec()); + let result = DbRow::new(db_json_entity, content); Ok(result) } @@ -231,9 +215,9 @@ impl DbJsonEntity { } pub fn restore_into_db_row( - json_first_line_reader: JsonFirstLineReader, + json_first_line_reader: JsonFirstLineIterator, ) -> Result { - let raw = json_first_line_reader.get_src_slice().to_vec(); + let raw = json_first_line_reader.as_slice().to_vec(); let db_row = Self::new(json_first_line_reader)?; let result = DbRow::new(db_row, raw); Ok(result) @@ -245,14 +229,12 @@ impl DbJsonEntity { ) -> Result>, DbEntityParseFail> { let mut result = Vec::new(); - let slice_iterator = SliceIterator::new(src); - - let json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIterator::new(src)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; let db_row = DbJsonEntity::parse_into_db_row( - json.unwrap_as_object(&json_array_iterator).unwrap(), + json.unwrap_as_object().unwrap(), inject_time_stamp, )?; result.push(Arc::new(db_row)); @@ -263,15 +245,11 @@ impl DbJsonEntity { pub fn restore_as_vec(src: &[u8]) -> Result>, DbEntityParseFail> { let mut result = Vec::new(); - let slice_iterator = SliceIterator::new(src); - - let json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIterator::new(src)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_entity = DbJsonEntity::restore_into_db_row( - json.unwrap_as_object(&json_array_iterator).unwrap(), - )?; + let db_entity = DbJsonEntity::restore_into_db_row(json.unwrap_as_object().unwrap())?; result.push(Arc::new(db_entity)); } return Ok(result); @@ -283,14 +261,12 @@ impl DbJsonEntity { ) -> Result>)>, DbEntityParseFail> { let mut result = Vec::new(); - let slice_iterator = SliceIterator::new(src); - - let json_array_iterator = JsonArrayIterator::new(slice_iterator)?; + let json_array_iterator = JsonArrayIterator::new(src)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; let db_row = DbJsonEntity::parse_into_db_row( - json.unwrap_as_object(&json_array_iterator).unwrap(), + json.unwrap_as_object().unwrap(), inject_time_stamp, )?; @@ -316,7 +292,7 @@ impl DbJsonEntity { ) -> Result>)>, DbEntityParseFail> { let mut result = Vec::new(); - let json_array_iterator = JsonArrayIteratorFromSlice::new(src)?; + let json_array_iterator = JsonArrayIterator::new(src)?; while let Some(json) = json_array_iterator.get_next() { let json = json?; @@ -443,10 +419,8 @@ pub fn get_the_end_of_the_json(data: &[u8]) -> usize { #[cfg(test)] mod tests { - use my_json::json_reader::JsonFirstLineReader; - use rust_extensions::{ - array_of_bytes_iterator::SliceIterator, date_time::DateTimeAsMicroseconds, - }; + use my_json::json_reader::{AsJsonSlice, JsonFirstLineIterator}; + use rust_extensions::date_time::DateTimeAsMicroseconds; use crate::db_json_entity::{DbEntityParseFail, JsonTimeStamp}; @@ -461,14 +435,13 @@ mod tests { "Expires": "2022-03-17T13:28:29.6537478Z" }"#; - let json_first_line_reader: JsonFirstLineReader = src_json.into(); + let json_first_line_reader = JsonFirstLineIterator::new(src_json.as_bytes()); let json_time = JsonTimeStamp::now(); let entity = DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_time).unwrap(); - let json_first_line_reader: JsonFirstLineReader = - entity.get_src_as_slice().into(); + let json_first_line_reader: JsonFirstLineIterator = entity.get_src_as_slice().into(); let dest_entity = DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_time).unwrap(); @@ -493,7 +466,7 @@ mod tests { "Expires": "2022-03-17T13:28:29.6537478Z" }"#; - let json_first_line_reader: JsonFirstLineReader> = src_json.into(); + let json_first_line_reader = JsonFirstLineIterator::new(src_json.as_bytes()); let entity = DbJsonEntity::new(json_first_line_reader).unwrap(); @@ -526,8 +499,7 @@ mod tests { "Expires": "2022-03-17T13:28:29.6537478Z" }"#; - let slice_iterator = SliceIterator::new(src_json.as_bytes()); - let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let json_first_line_reader = JsonFirstLineIterator::new(src_json.as_bytes()); let result = DbJsonEntity::new(json_first_line_reader); @@ -541,8 +513,8 @@ mod tests { let src_json = r#"{"value":{"is_enabled":true,"fee_percent":5.0,"min_balance_usd":100.0,"fee_period_days":30,"inactivity_period_days":90},"PartitionKey":"*","RowKey":"*"}"#; let time_stamp = JsonTimeStamp::now(); - let slice_iterator = SliceIterator::new(src_json.as_bytes()); - let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + + let json_first_line_reader = JsonFirstLineIterator::new(src_json.as_bytes()); let db_row = DbJsonEntity::parse_into_db_row(json_first_line_reader, &time_stamp).unwrap(); println!( @@ -559,8 +531,7 @@ mod tests { let mut json = r#"{"PartitionKey":"PK", "RowKey":"RK"} "#.as_bytes().to_vec(); - let slice_iterator = SliceIterator::new(json.as_slice()); - let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let json_first_line_reader = JsonFirstLineIterator::new(json.as_slice()); let mut db_json_entity = DbJsonEntity::new(json_first_line_reader).unwrap(); @@ -591,8 +562,7 @@ mod tests { let json = r#"{"PartitionKey":"Pk", "RowKey":"Rk", "timestamp":null}"#; - let slice_iterator = SliceIterator::new(json.as_bytes()); - let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let json_first_line_reader = JsonFirstLineIterator::new(json.as_slice()); let db_row = DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_ts).unwrap(); @@ -608,8 +578,7 @@ mod tests { let json = r#"{"PartitionKey":"Pk", "RowKey":"Rk", "timestamp":"12345678901234567890123456789012345678901234567890"}"#; - let slice_iterator = SliceIterator::new(json.as_bytes()); - let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let json_first_line_reader = JsonFirstLineIterator::new(json.as_bytes()); let db_json_entity = DbJsonEntity::parse_into_db_row(json_first_line_reader, &json_ts).unwrap(); @@ -630,8 +599,7 @@ mod tests { let inject_time_stamp = JsonTimeStamp::now(); - let slice_iterator = SliceIterator::new(test_json.as_bytes()); - let json_first_line_reader = JsonFirstLineReader::new(slice_iterator); + let json_first_line_reader = JsonFirstLineIterator::new(test_json.as_bytes()); let db_row = DbJsonEntity::parse_into_db_row(json_first_line_reader, &inject_time_stamp).unwrap(); diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs b/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs index b83a7a0..e98ec23 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity_with_content.rs @@ -1,5 +1,4 @@ -use my_json::json_reader::JsonFirstLineReader; -use rust_extensions::array_of_bytes_iterator::SliceIterator; +use my_json::json_reader::JsonFirstLineIterator; use crate::db::DbRow; @@ -37,8 +36,7 @@ impl<'s> DbJsonEntityWithContent<'s> { } pub fn into_db_row(self) -> Result { - let slice_iterator = SliceIterator::new(self.raw); - let first_line_reader = JsonFirstLineReader::new(slice_iterator); + let first_line_reader = JsonFirstLineIterator::new(self.raw); DbJsonEntity::parse_into_db_row(first_line_reader, &self.time_stamp) } } diff --git a/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs b/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs index 4b8817e..db701cd 100644 --- a/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs +++ b/my-no-sql-core/src/db_json_entity/db_row_content_compiler/db_row_content_compiler.rs @@ -1,4 +1,4 @@ -use my_json::json_reader::JsonKeyValueRef; +use my_json::json_reader::{JsonFieldNameRef, JsonValueRef}; use crate::db_json_entity::{JsonKeyValuePosition, KeyValueContentPosition}; @@ -37,15 +37,18 @@ impl DbRowContentCompiler { } } - pub fn append(&mut self, line: JsonKeyValueRef) -> JsonKeyValuePosition { + pub fn append( + &mut self, + name: &JsonFieldNameRef, + the_value: &JsonValueRef, + ) -> JsonKeyValuePosition { self.append_first_line(); let mut key = KeyValueContentPosition { start: self.content.len(), end: 0, }; - self.content - .extend_from_slice(line.name.as_raw_str().unwrap().as_bytes()); + self.content.extend_from_slice(name.as_slice()); key.end = self.content.len(); @@ -55,7 +58,7 @@ impl DbRowContentCompiler { start: self.content.len(), end: 0, }; - self.content.extend_from_slice(line.value.as_bytes()); + self.content.extend_from_slice(the_value.as_slice()); value.end = self.content.len(); diff --git a/my-no-sql-core/src/db_json_entity/json_key_value_position.rs b/my-no-sql-core/src/db_json_entity/json_key_value_position.rs index 91574ee..648365d 100644 --- a/my-no-sql-core/src/db_json_entity/json_key_value_position.rs +++ b/my-no-sql-core/src/db_json_entity/json_key_value_position.rs @@ -1,4 +1,4 @@ -use my_json::json_reader::JsonKeyValue; +use my_json::json_reader::{JsonContentOffset, JsonValue}; #[derive(Debug, Clone)] pub struct KeyValueContentPosition { @@ -31,16 +31,16 @@ pub struct JsonKeyValuePosition { } impl JsonKeyValuePosition { - pub fn new(src: &JsonKeyValue) -> Self { + pub fn new(name: &JsonContentOffset, value: &JsonValue) -> Self { Self { key: KeyValueContentPosition { - start: src.name.start, - end: src.name.end, + start: name.start, + end: name.end, }, value: KeyValueContentPosition { - start: src.value.start, - end: src.value.end, + start: value.start, + end: value.end, }, } } diff --git a/my-no-sql-core/src/entity_serializer.rs b/my-no-sql-core/src/entity_serializer.rs index ee2d930..4f2fec2 100644 --- a/my-no-sql-core/src/entity_serializer.rs +++ b/my-no-sql-core/src/entity_serializer.rs @@ -1,6 +1,5 @@ -use my_json::json_reader::JsonFirstLineReader; +use my_json::json_reader::JsonFirstLineIterator; use my_no_sql_abstractions::MyNoSqlEntity; -use rust_extensions::array_of_bytes_iterator::SliceIterator; use serde::{de::DeserializeOwned, Serialize}; use crate::db_json_entity::DbJsonEntity; @@ -21,8 +20,8 @@ where match parse_result { Ok(el) => return Ok(el), Err(err) => { - let slice_iterator = SliceIterator::new(data); - let json_first_line_iterator = JsonFirstLineReader::new(slice_iterator); + + let json_first_line_iterator = JsonFirstLineIterator::new(data); let db_entity = DbJsonEntity::new(json_first_line_iterator); match db_entity { diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 7484425..93b8edf 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -1,11 +1,10 @@ use flurl::{FlUrl, FlUrlResponse}; use my_json::{ - json_reader::array_iterator::JsonArrayIterator, + json_reader::JsonArrayIterator, json_writer::{JsonArrayWriter, RawJsonObject}, }; use my_logger::LogEventCtx; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; -use rust_extensions::array_of_bytes_iterator::SliceIterator; use serde::{Deserialize, Serialize}; use crate::{CreateTableParams, DataWriterError, OperationFailHttpContract, UpdateReadStatistics}; @@ -549,8 +548,8 @@ fn deserialize_entities( src: &[u8], ) -> Result, DataWriterError> { let mut result = Vec::new(); - let slice_iterator = SliceIterator::new(src); - let json_array_iterator = JsonArrayIterator::new(slice_iterator); + + let json_array_iterator = JsonArrayIterator::new(src); if let Err(err) = &json_array_iterator { panic!( @@ -560,12 +559,12 @@ fn deserialize_entities( ); } - let mut json_array_iterator = json_array_iterator.unwrap(); + let json_array_iterator = json_array_iterator.unwrap(); while let Some(item) = json_array_iterator.get_next() { let itm = item.unwrap(); - result.push(TEntity::deserialize_entity(itm.as_bytes(&json_array_iterator)).unwrap()); + result.push(TEntity::deserialize_entity(itm.as_bytes()).unwrap()); } Ok(result) diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 1464af6..ec65c3c 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -1,10 +1,10 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration}; use async_trait::async_trait; -use my_json::json_reader::array_iterator::JsonArrayIterator; +use my_json::json_reader::JsonArrayIterator; use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; use my_no_sql_tcp_shared::sync_to_main::SyncToMainNodeHandler; -use rust_extensions::{array_of_bytes_iterator::SliceIterator, ApplicationStates, StrOrString}; +use rust_extensions::{ApplicationStates, StrOrString}; use serde::de::DeserializeOwned; use tokio::sync::Mutex; @@ -137,9 +137,7 @@ where &self, data: &[u8], ) -> BTreeMap>> { - let slice_iterator = SliceIterator::new(data); - - let json_array_iterator = JsonArrayIterator::new(slice_iterator); + let json_array_iterator = JsonArrayIterator::new(data); if let Err(err) = &json_array_iterator { panic!( @@ -149,7 +147,7 @@ where ); } - let mut json_array_iterator = json_array_iterator.unwrap(); + let json_array_iterator = json_array_iterator.unwrap(); let mut result = BTreeMap::new(); while let Some(db_entity) = json_array_iterator.get_next() { @@ -164,7 +162,7 @@ where let db_entity_data = db_entity.unwrap(); let item_to_insert = if TMyNoSqlEntity::LAZY_DESERIALIZATION { - let data = db_entity_data.as_bytes(&json_array_iterator).to_vec(); + let data = db_entity_data.as_bytes().to_vec(); let db_json_entity = my_no_sql_core::db_json_entity::DbJsonEntity::from_slice(&data).unwrap(); @@ -177,11 +175,9 @@ where ) } else { LazyMyNoSqlEntity::Deserialized( - TMyNoSqlEntity::deserialize_entity( - db_entity_data.as_bytes(&json_array_iterator), - ) - .unwrap() - .into(), + TMyNoSqlEntity::deserialize_entity(db_entity_data.as_bytes()) + .unwrap() + .into(), ) }; diff --git a/my-no-sql-tcp-reader/src/tcp_events.rs b/my-no-sql-tcp-reader/src/tcp_events.rs index a4fec08..e532b7d 100644 --- a/my-no-sql-tcp-reader/src/tcp_events.rs +++ b/my-no-sql-tcp-reader/src/tcp_events.rs @@ -3,13 +3,12 @@ use std::sync::Arc; use my_no_sql_tcp_shared::{ sync_to_main::SyncToMainNodeHandler, MyNoSqlReaderTcpSerializer, MyNoSqlTcpContract, }; -use my_tcp_sockets::{ - tcp_connection::TcpSocketConnection, SocketEventCallback, TcpSerializerState, -}; +use my_tcp_sockets::{tcp_connection::TcpSocketConnection, SocketEventCallback}; use crate::subscribers::Subscribers; -pub type TcpConnection = TcpSocketConnection; +pub type MyNoSqlTcpConnection = + TcpSocketConnection; pub struct TcpEvents { app_name: String, pub subscribers: Subscribers, @@ -28,10 +27,7 @@ impl TcpEvents { #[async_trait::async_trait] impl SocketEventCallback for TcpEvents { - async fn connected( - &self, - connection: Arc>, - ) { + async fn connected(&self, connection: Arc) { let contract = MyNoSqlTcpContract::Greeting { name: self.app_name.to_string(), }; @@ -50,19 +46,12 @@ impl SocketEventCallback for .tcp_events_pusher_new_connection_established(connection); } - async fn disconnected( - &self, - connection: Arc>, - ) { + async fn disconnected(&self, connection: Arc) { self.sync_handler .tcp_events_pusher_connection_disconnected(connection); } - async fn payload( - &self, - _connection: &Arc>, - contract: MyNoSqlTcpContract, - ) { + async fn payload(&self, _connection: &Arc, contract: MyNoSqlTcpContract) { match contract { MyNoSqlTcpContract::Ping => {} MyNoSqlTcpContract::Pong => {} diff --git a/my-no-sql-tcp-shared/src/tcp_contracts.rs b/my-no-sql-tcp-shared/src/tcp_contracts.rs index f45020e..e73c7c0 100644 --- a/my-no-sql-tcp-shared/src/tcp_contracts.rs +++ b/my-no-sql-tcp-shared/src/tcp_contracts.rs @@ -498,9 +498,9 @@ impl my_tcp_sockets::TcpContract for MyNoSqlTcpContract { } impl TcpSerializerState for () { - fn is_tcp_contract_related_to_metadata(&self, contract: &MyNoSqlTcpContract) -> bool { + fn is_tcp_contract_related_to_metadata(&self, _contract: &MyNoSqlTcpContract) -> bool { false } - fn apply_tcp_contract(&mut self, contract: &MyNoSqlTcpContract) {} + fn apply_tcp_contract(&mut self, _contract: &MyNoSqlTcpContract) {} } From a070f21a1f455a485c12f9a1daef547e1d0dbad8 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 20 Nov 2024 00:31:40 +0200 Subject: [PATCH 40/89] Removed debug info --- my-no-sql-core/src/db_json_entity/db_json_entity.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity.rs b/my-no-sql-core/src/db_json_entity/db_json_entity.rs index ee7dc91..e948041 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity.rs @@ -124,13 +124,10 @@ impl DbJsonEntity { match name { super::consts::PARTITION_KEY => { partition_key = Some(raw.append(&name_ref, &value_ref)); - - println!("PartitionKey: {:?}", partition_key); } super::consts::ROW_KEY => { row_key = Some(raw.append(&name_ref, &value_ref)); - println!("RowKey: {:?}", row_key); time_stamp = raw .append_str_value(super::consts::TIME_STAMP, now.as_str()) .into(); From 81c8dba04cf2d3bbd39c012cea414aa40bd68b61 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 20 Nov 2024 18:02:38 +0200 Subject: [PATCH 41/89] DbTableName now is a structure --- my-no-sql-core/src/db/db_table/db_table.rs | 16 +++++++++----- .../src/db/db_table/db_table_master_node.rs | 14 ++++--------- .../src/db/db_table/db_table_name.rs | 21 +++++++++++++++++++ my-no-sql-core/src/db/db_table/mod.rs | 2 ++ 4 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 my-no-sql-core/src/db/db_table/db_table_name.rs diff --git a/my-no-sql-core/src/db/db_table/db_table.rs b/my-no-sql-core/src/db/db_table/db_table.rs index c695cc0..1f529a3 100644 --- a/my-no-sql-core/src/db/db_table/db_table.rs +++ b/my-no-sql-core/src/db/db_table/db_table.rs @@ -1,17 +1,17 @@ use my_json::json_writer::JsonArrayWriter; #[cfg(feature = "master-node")] use rust_extensions::date_time::DateTimeAsMicroseconds; -use rust_extensions::sorted_vec::SortedVecWithStrKey; +use rust_extensions::sorted_vec::{EntityWithStrKey, SortedVecWithStrKey}; use std::sync::Arc; use crate::db::{DbPartition, DbRow, PartitionKey, PartitionKeyParameter, RowKeyParameter}; #[cfg(feature = "master-node")] use super::DbTableAttributes; -use super::{AllDbRowsIterator, AvgSize, ByRowKeyIterator, DbPartitionsContainer}; +use super::{AllDbRowsIterator, AvgSize, ByRowKeyIterator, DbPartitionsContainer, DbTableName}; pub struct DbTable { - pub name: String, + pub name: DbTableName, pub partitions: DbPartitionsContainer, pub avg_size: AvgSize, #[cfg(feature = "master-node")] @@ -20,11 +20,17 @@ pub struct DbTable { pub attributes: DbTableAttributes, } +impl EntityWithStrKey for DbTable { + fn get_key(&self) -> &str { + self.name.as_str() + } +} + impl DbTable { #[cfg(not(feature = "master-node"))] - pub fn new(name: String) -> Self { + pub fn new(name: DbTableName) -> Self { Self { - name: name.into(), + name, partitions: DbPartitionsContainer::new(), avg_size: AvgSize::new(), } diff --git a/my-no-sql-core/src/db/db_table/db_table_master_node.rs b/my-no-sql-core/src/db/db_table/db_table_master_node.rs index 0633795..87f0b88 100644 --- a/my-no-sql-core/src/db/db_table/db_table_master_node.rs +++ b/my-no-sql-core/src/db/db_table/db_table_master_node.rs @@ -5,7 +5,7 @@ use rust_extensions::{ use crate::db::PartitionKey; -use super::{AvgSize, DataToGc, DbPartitionsContainer, DbTable, DbTableAttributes}; +use super::{AvgSize, DataToGc, DbPartitionsContainer, DbTable, DbTableAttributes, DbTableName}; pub struct PartitionLastWriteMoment { pub partition_key: PartitionKey, @@ -19,7 +19,7 @@ impl EntityWithStrKey for PartitionLastWriteMoment { } impl DbTable { - pub fn new(name: String, attributes: DbTableAttributes) -> Self { + pub fn new(name: DbTableName, attributes: DbTableAttributes) -> Self { Self { name, partitions: DbPartitionsContainer::new(), @@ -113,10 +113,7 @@ mod tests { #[test] fn test_insert_record() { - let mut db_table = DbTable::new( - "test-table".to_string(), - DbTableAttributes::create_default(), - ); + let mut db_table = DbTable::new("test-table".into(), DbTableAttributes::create_default()); let now = JsonTimeStamp::now(); @@ -137,10 +134,7 @@ mod tests { #[test] fn test_insert_and_insert_or_replace() { - let mut db_table = DbTable::new( - "test-table".to_string(), - DbTableAttributes::create_default(), - ); + let mut db_table = DbTable::new("test-table".into(), DbTableAttributes::create_default()); let now = JsonTimeStamp::now(); diff --git a/my-no-sql-core/src/db/db_table/db_table_name.rs b/my-no-sql-core/src/db/db_table/db_table_name.rs new file mode 100644 index 0000000..1d5fc72 --- /dev/null +++ b/my-no-sql-core/src/db/db_table/db_table_name.rs @@ -0,0 +1,21 @@ +use std::sync::Arc; + +pub struct DbTableName(Arc); + +impl DbTableName { + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl Into for String { + fn into(self) -> DbTableName { + DbTableName(Arc::new(self)) + } +} + +impl<'s> Into for &'s str { + fn into(self) -> DbTableName { + DbTableName(Arc::new(self.to_string())) + } +} diff --git a/my-no-sql-core/src/db/db_table/mod.rs b/my-no-sql-core/src/db/db_table/mod.rs index f64e387..8c82ee8 100644 --- a/my-no-sql-core/src/db/db_table/mod.rs +++ b/my-no-sql-core/src/db/db_table/mod.rs @@ -27,3 +27,5 @@ mod all_db_rows_iterator; pub use all_db_rows_iterator::*; mod by_row_key_iterator; pub use by_row_key_iterator::*; +mod db_table_name; +pub use db_table_name::*; From 59c656391f9749be8ef3f2f97f1d2d7399df7fde Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 20 Nov 2024 18:05:49 +0200 Subject: [PATCH 42/89] Improvements --- my-no-sql-core/src/db/db_table/db_table_name.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/my-no-sql-core/src/db/db_table/db_table_name.rs b/my-no-sql-core/src/db/db_table/db_table_name.rs index 1d5fc72..592ea6e 100644 --- a/my-no-sql-core/src/db/db_table/db_table_name.rs +++ b/my-no-sql-core/src/db/db_table/db_table_name.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct DbTableName(Arc); impl DbTableName { From ea41fd2cf05fac3f3e1b2b88ec4d0c9fe043d535 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 21 Nov 2024 10:04:35 +0200 Subject: [PATCH 43/89] Improvements --- my-no-sql-core/src/db/db_table/db_table_name.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/my-no-sql-core/src/db/db_table/db_table_name.rs b/my-no-sql-core/src/db/db_table/db_table_name.rs index 592ea6e..46f4d33 100644 --- a/my-no-sql-core/src/db/db_table/db_table_name.rs +++ b/my-no-sql-core/src/db/db_table/db_table_name.rs @@ -7,6 +7,10 @@ impl DbTableName { pub fn as_str(&self) -> &str { self.0.as_str() } + + pub fn to_string(&self) -> String { + self.0.to_string() + } } impl Into for String { From 4f90eb2cd1bb488db004463bb8f4ce58f2e14f51 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 21 Nov 2024 10:09:45 +0200 Subject: [PATCH 44/89] Improvements --- my-no-sql-core/src/db/db_table/db_table_name.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/my-no-sql-core/src/db/db_table/db_table_name.rs b/my-no-sql-core/src/db/db_table/db_table_name.rs index 46f4d33..d7d85f6 100644 --- a/my-no-sql-core/src/db/db_table/db_table_name.rs +++ b/my-no-sql-core/src/db/db_table/db_table_name.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{fmt::Display, sync::Arc}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct DbTableName(Arc); @@ -13,6 +13,12 @@ impl DbTableName { } } +impl Display for DbTableName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + impl Into for String { fn into(self) -> DbTableName { DbTableName(Arc::new(self)) From 53626b5528b2138872e4e2e7b57a8aa5a7a07e38 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 21 Nov 2024 16:48:42 +0200 Subject: [PATCH 45/89] Improved the way restoration is done --- my-no-sql-core/src/db_json_entity/db_json_entity.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/my-no-sql-core/src/db_json_entity/db_json_entity.rs b/my-no-sql-core/src/db_json_entity/db_json_entity.rs index e948041..d2e57c4 100644 --- a/my-no-sql-core/src/db_json_entity/db_json_entity.rs +++ b/my-no-sql-core/src/db_json_entity/db_json_entity.rs @@ -211,10 +211,8 @@ impl DbJsonEntity { None } - pub fn restore_into_db_row( - json_first_line_reader: JsonFirstLineIterator, - ) -> Result { - let raw = json_first_line_reader.as_slice().to_vec(); + pub fn restore_into_db_row(raw: Vec) -> Result { + let json_first_line_reader = JsonFirstLineIterator::new(raw.as_slice()); let db_row = Self::new(json_first_line_reader)?; let result = DbRow::new(db_row, raw); Ok(result) @@ -246,7 +244,7 @@ impl DbJsonEntity { while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_entity = DbJsonEntity::restore_into_db_row(json.unwrap_as_object().unwrap())?; + let db_entity = DbJsonEntity::restore_into_db_row(json.as_bytes().to_vec())?; result.push(Arc::new(db_entity)); } return Ok(result); @@ -293,7 +291,7 @@ impl DbJsonEntity { while let Some(json) = json_array_iterator.get_next() { let json = json?; - let db_row = DbJsonEntity::restore_into_db_row(json.unwrap_as_object().unwrap())?; + let db_row = DbJsonEntity::restore_into_db_row(json.as_bytes().to_vec())?; let partition_key = db_row.get_partition_key(); From 14cc45eea81f548a08c89a52b7b6bf228352f1a7 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 21 Nov 2024 16:57:31 +0200 Subject: [PATCH 46/89] Made restore method for a partition --- my-no-sql-core/src/db/db_table/db_table.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/my-no-sql-core/src/db/db_table/db_table.rs b/my-no-sql-core/src/db/db_table/db_table.rs index 1f529a3..35203ae 100644 --- a/my-no-sql-core/src/db/db_table/db_table.rs +++ b/my-no-sql-core/src/db/db_table/db_table.rs @@ -36,6 +36,10 @@ impl DbTable { } } + pub fn restore_partition(&mut self, db_partition: DbPartition) { + self.partitions.insert(db_partition); + } + pub fn get_partitions_amount(&self) -> usize { self.partitions.len() } From 2dd64329c2c21b9485e2daab38158372b186323c Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 22 Nov 2024 18:13:44 +0200 Subject: [PATCH 47/89] Fixes --- .../my_no_sql_data_writer.rs | 1 - .../src/my_no_sql_data_writer/with_retries.rs | 36 +++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 18bcd75..d605d3e 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -133,7 +133,6 @@ impl MyNoSqlData MyNoSqlDataWriterWithRetries::new( self.fl_url_factory.clone(), self.sync_period, - delay_between_attempts, max_attempts, ) } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs index 97c8874..76b28b3 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/with_retries.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, time::Duration}; +use std::marker::PhantomData; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; @@ -9,7 +9,6 @@ use super::fl_url_factory::FlUrlFactory; pub struct MyNoSqlDataWriterWithRetries { fl_url_factory: FlUrlFactory, sync_period: DataSynchronizationPeriod, - attempt_delay: Duration, phantom: PhantomData, max_attempts: usize, } @@ -20,13 +19,12 @@ impl pub fn new( fl_url_factory: FlUrlFactory, sync_period: DataSynchronizationPeriod, - attempt_delay: Duration, max_attempts: usize, ) -> Self { Self { phantom: PhantomData, sync_period, - attempt_delay, + max_attempts, fl_url_factory, } @@ -34,13 +32,13 @@ impl pub async fn insert_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::insert_entity(fl_url, entity, &self.sync_period).await } pub async fn insert_or_replace_entity(&self, entity: &TEntity) -> Result<(), DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::insert_or_replace_entity(fl_url, entity, &self.sync_period).await } @@ -49,7 +47,7 @@ impl entities: &[TEntity], ) -> Result<(), DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::bulk_insert_or_replace(fl_url, entities, &self.sync_period).await } @@ -60,7 +58,7 @@ impl update_read_statistics: Option, ) -> Result, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::get_entity( fl_url, partition_key, @@ -76,7 +74,7 @@ impl update_read_statistics: Option, ) -> Result>, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::get_by_partition_key( fl_url, partition_key, @@ -97,7 +95,7 @@ impl update_read_statistics: Option, ) -> Result>, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::get_enum_case_models_by_partition_key( fl_url, update_read_statistics.as_ref(), @@ -117,7 +115,7 @@ impl update_read_statistics: Option, ) -> Result, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::get_enum_case_model(fl_url, update_read_statistics.as_ref()).await } @@ -126,7 +124,7 @@ impl row_key: &str, ) -> Result>, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::get_by_row_key(fl_url, row_key).await } @@ -141,7 +139,7 @@ impl &self, ) -> Result, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::delete_enum_case(fl_url).await } @@ -157,7 +155,7 @@ impl row_key: &str, ) -> Result, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::delete_enum_case_with_row_key(fl_url, row_key).await } @@ -167,19 +165,19 @@ impl row_key: &str, ) -> Result, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::delete_row(fl_url, partition_key, row_key).await } pub async fn delete_partitions(&self, partition_keys: &[&str]) -> Result<(), DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::delete_partitions(fl_url, TEntity::TABLE_NAME, partition_keys).await } pub async fn get_all(&self) -> Result>, DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::get_all(fl_url).await } @@ -188,7 +186,7 @@ impl entities: &[TEntity], ) -> Result<(), DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::clean_table_and_bulk_insert(fl_url, entities, &self.sync_period).await } @@ -198,7 +196,7 @@ impl entities: &[TEntity], ) -> Result<(), DataWriterError> { let (fl_url, _) = self.fl_url_factory.get_fl_url().await?; - let fl_url = fl_url.with_retries(self.max_attempts, self.attempt_delay); + let fl_url = fl_url.with_retries(self.max_attempts); super::execution::clean_partition_and_bulk_insert( fl_url, partition_key, From 9f910d521a7ca6bcac4d818fa44ed2dbfd22fa16 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 22 Nov 2024 23:37:41 +0200 Subject: [PATCH 48/89] Fixes --- my-no-sql-data-writer/src/ping_pool.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/my-no-sql-data-writer/src/ping_pool.rs b/my-no-sql-data-writer/src/ping_pool.rs index 4241407..67401d9 100644 --- a/my-no-sql-data-writer/src/ping_pool.rs +++ b/my-no-sql-data-writer/src/ping_pool.rs @@ -117,15 +117,6 @@ async fn ping_loop() { println!("{}:{} ping error: {:?}", itm.name, itm.version, err); continue; } - - let fl_url_response = fl_url_response.unwrap(); - - println!( - "{}:{} ping status code {}", - itm.name, - itm.version, - fl_url_response.get_status_code() - ) } } } From 9605604c1eb8e4d16b9bcfc1fdaf65ff1e7e6107 Mon Sep 17 00:00:00 2001 From: amigin Date: Sun, 24 Nov 2024 16:04:09 +0200 Subject: [PATCH 49/89] Removed Delay between requests --- .../src/my_no_sql_data_writer/my_no_sql_data_writer.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index d605d3e..33de38d 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, sync::Arc, time::Duration}; +use std::{marker::PhantomData, sync::Arc}; use flurl::FlUrl; @@ -125,11 +125,7 @@ impl MyNoSqlData .await } - pub fn with_retries( - &self, - delay_between_attempts: Duration, - max_attempts: usize, - ) -> MyNoSqlDataWriterWithRetries { + pub fn with_retries(&self, max_attempts: usize) -> MyNoSqlDataWriterWithRetries { MyNoSqlDataWriterWithRetries::new( self.fl_url_factory.clone(), self.sync_period, From cfdfc14852de300b2d3902cccb66addf31d3c92f Mon Sep 17 00:00:00 2001 From: amigin Date: Sun, 24 Nov 2024 16:05:26 +0200 Subject: [PATCH 50/89] Removed delay between retries since we do not need it anymore --- my-no-sql-data-writer/src/ping_pool.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/my-no-sql-data-writer/src/ping_pool.rs b/my-no-sql-data-writer/src/ping_pool.rs index 67401d9..72b4b6f 100644 --- a/my-no-sql-data-writer/src/ping_pool.rs +++ b/my-no-sql-data-writer/src/ping_pool.rs @@ -108,6 +108,7 @@ async fn ping_loop() { let fl_url_response = fl_url .unwrap() .0 + .with_retries(3) .append_path_segment("api") .append_path_segment("ping") .post_json(&ping_model) From c2d65640d4013b0ea6de53d0a29069786aef891b Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 21 Dec 2024 17:54:55 +0200 Subject: [PATCH 51/89] Fixes --- my-no-sql-core/src/db/db_partition/db_partition.rs | 8 ++++---- my-no-sql-core/src/db/db_row/db_row.rs | 11 ++++++----- .../src/my_no_sql_data_writer/execution.rs | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/my-no-sql-core/src/db/db_partition/db_partition.rs b/my-no-sql-core/src/db/db_partition/db_partition.rs index 98077f8..1b88533 100644 --- a/my-no-sql-core/src/db/db_partition/db_partition.rs +++ b/my-no-sql-core/src/db/db_partition/db_partition.rs @@ -201,17 +201,17 @@ impl DbPartition { } impl JsonObject for &'_ DbPartition { - fn write_into(&self, dest: &mut Vec) { + fn write_into(&self, dest: &mut String) { let mut first_element = true; - dest.push(b'['); + dest.push('['); for db_row in self.rows.get_all() { if first_element { first_element = false; } else { - dest.push(b','); + dest.push(','); } db_row.as_ref().write_into(dest) } - dest.push(b']'); + dest.push(']'); } } diff --git a/my-no-sql-core/src/db/db_row/db_row.rs b/my-no-sql-core/src/db/db_row/db_row.rs index 65a1862..dd193e8 100644 --- a/my-no-sql-core/src/db/db_row/db_row.rs +++ b/my-no-sql-core/src/db/db_row/db_row.rs @@ -160,14 +160,15 @@ impl DbRow { } #[cfg(not(feature = "master-node"))] - pub fn write_json(&self, out: &mut Vec) { - out.extend_from_slice(&self.raw); + pub fn write_json(&self, out: &mut String) { + let str = unsafe { std::str::from_utf8_unchecked(&self.raw) }; + out.push_str(str); } pub fn to_vec(&self) -> Vec { - let mut result = Vec::new(); + let mut result = String::new(); self.write_json(&mut result); - result + result.into_bytes() } } @@ -247,7 +248,7 @@ fn find_json_separator_after(src: &[u8], pos: usize) -> Option { } impl JsonObject for &'_ DbRow { - fn write_into(&self, dest: &mut Vec) { + fn write_into(&self, dest: &mut String) { self.write_json(dest) } } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 93b8edf..db8a834 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -493,7 +493,7 @@ fn serialize_entities_to_body( json_array_writer.write(payload); } - Some(json_array_writer.build()) + Some(json_array_writer.build().into_bytes()) } async fn check_error(response: &mut FlUrlResponse) -> Result<(), DataWriterError> { From a31971cdad2b380e3f21e9b330c595be51436e21 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 24 Dec 2024 00:24:07 +0200 Subject: [PATCH 52/89] Improvements with SSH --- .../src/db/db_partition/db_partition.rs | 4 +-- .../src/db/db_partition/db_rows_container.rs | 2 +- my-no-sql-data-writer/src/lib.rs | 3 -- .../my_no_sql_data_writer/fl_url_factory.rs | 24 +++++++------- .../my_no_sql_data_writer.rs | 31 +++++-------------- my-no-sql-data-writer/src/ssh.rs | 8 ----- .../src/subscribers/my_no_sql_data_reader.rs | 2 +- .../subscribers/my_no_sql_data_reader_tcp.rs | 4 +-- 8 files changed, 24 insertions(+), 54 deletions(-) delete mode 100644 my-no-sql-data-writer/src/ssh.rs diff --git a/my-no-sql-core/src/db/db_partition/db_partition.rs b/my-no-sql-core/src/db/db_partition/db_partition.rs index 1b88533..0d15cb1 100644 --- a/my-no-sql-core/src/db/db_partition/db_partition.rs +++ b/my-no-sql-core/src/db/db_partition/db_partition.rs @@ -132,11 +132,11 @@ impl DbPartition { result.get_result() } - pub fn get_all_rows<'s>(&'s self) -> std::slice::Iter> { + pub fn get_all_rows(&self) -> std::slice::Iter> { self.rows.get_all() } - pub fn get_all_rows_cloned<'s>(&'s self) -> Vec> { + pub fn get_all_rows_cloned(&self) -> Vec> { self.rows.get_all().map(|itm| itm.clone()).collect() } diff --git a/my-no-sql-core/src/db/db_partition/db_rows_container.rs b/my-no-sql-core/src/db/db_partition/db_rows_container.rs index 95f7170..810e2fd 100644 --- a/my-no-sql-core/src/db/db_partition/db_rows_container.rs +++ b/my-no-sql-core/src/db/db_partition/db_rows_container.rs @@ -100,7 +100,7 @@ impl DbRowsContainer { return self.data.contains(row_key); } - pub fn get_all<'s>(&'s self) -> std::slice::Iter> { + pub fn get_all(&self) -> std::slice::Iter> { self.data.iter() } diff --git a/my-no-sql-data-writer/src/lib.rs b/my-no-sql-data-writer/src/lib.rs index d1ad64e..5f0b2ce 100644 --- a/my-no-sql-data-writer/src/lib.rs +++ b/my-no-sql-data-writer/src/lib.rs @@ -1,9 +1,6 @@ mod my_no_sql_data_writer; pub use my_no_sql_data_writer::*; -#[cfg(feature = "with-ssh")] -mod ssh; - mod ping_pool; pub use ping_pool::*; diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index 1944d20..1869681 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -2,9 +2,6 @@ use std::sync::Arc; use flurl::FlUrl; -#[cfg(feature = "with-ssh")] -use flurl::my_ssh::SshSessionsPool; - use rust_extensions::UnsafeValue; use super::{CreateTableParams, DataWriterError, MyNoSqlWriterSettings}; @@ -13,12 +10,10 @@ use super::{CreateTableParams, DataWriterError, MyNoSqlWriterSettings}; pub struct FlUrlFactory { settings: Arc, auto_create_table_params: Option>, + #[cfg(feature = "with-ssh")] - pub ssh_sessions_pool: Option>, - #[cfg(feature = "with-ssh")] - pub http_buffer_size: Option, - #[cfg(feature = "with-ssh")] - pub ssh_credentials: Arc, + pub ssh_security_credentials_resolver: + Option>, create_table_is_called: Arc>, table_name: &'static str, @@ -32,21 +27,24 @@ impl FlUrlFactory { ) -> Self { Self { auto_create_table_params, - #[cfg(feature = "with-ssh")] - ssh_sessions_pool: None, + create_table_is_called: UnsafeValue::new(false).into(), settings, table_name, #[cfg(feature = "with-ssh")] - http_buffer_size: None, - #[cfg(feature = "with-ssh")] - ssh_credentials: Arc::new(crate::ssh::SshCredentials::UserAgent), + ssh_security_credentials_resolver: None, } } async fn create_fl_url(&self, url: &str) -> FlUrl { let fl_url = flurl::FlUrl::new(url); + + #[cfg(feature = "with-ssh")] + if let Some(ssh_security_credentials_resolver) = &self.ssh_security_credentials_resolver { + return fl_url.set_ssh_private_key_resolver(ssh_security_credentials_resolver.clone()); + } + fl_url } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs index 33de38d..de2e743 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/my_no_sql_data_writer.rs @@ -4,9 +4,6 @@ use flurl::FlUrl; use my_no_sql_abstractions::{DataSynchronizationPeriod, MyNoSqlEntity, MyNoSqlEntitySerializer}; -#[cfg(feature = "with-ssh")] -use flurl::my_ssh::*; - use serde::{Deserialize, Serialize}; use crate::{MyNoSqlDataWriterWithRetries, MyNoSqlWriterSettings}; @@ -87,27 +84,13 @@ impl MyNoSqlData } #[cfg(feature = "with-ssh")] - pub fn set_ssh_password(&mut self, password: String) { - self.fl_url_factory.ssh_credentials = - Arc::new(crate::ssh::SshCredentials::Password(password)); - } - - #[cfg(feature = "with-ssh")] - pub fn set_ssh_private_key(&mut self, private_key: String, passphrase: String) { - self.fl_url_factory.ssh_credentials = Arc::new(crate::ssh::SshCredentials::PrivateKey { - private_key, - passphrase, - }); - } - - #[cfg(feature = "with-ssh")] - pub fn set_ssh_sessions_pool(&mut self, ssh_sessions_pool: Arc) { - self.fl_url_factory.ssh_sessions_pool = Some(ssh_sessions_pool); - } - - #[cfg(feature = "with-ssh")] - pub fn set_http_buffer_size(&mut self, buffer_size: usize) { - self.fl_url_factory.http_buffer_size = Some(buffer_size); + pub fn set_ssh_security_credentials_resolver( + &mut self, + resolver: Arc< + dyn flurl::my_ssh::ssh_settings::SshSecurityCredentialsResolver + Send + Sync, + >, + ) { + self.fl_url_factory.ssh_security_credentials_resolver = Some(resolver); } pub async fn create_table_if_not_exists( diff --git a/my-no-sql-data-writer/src/ssh.rs b/my-no-sql-data-writer/src/ssh.rs deleted file mode 100644 index 2cc914e..0000000 --- a/my-no-sql-data-writer/src/ssh.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub enum SshCredentials { - UserAgent, - Password(String), - PrivateKey { - private_key: String, - passphrase: String, - }, -} diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs index 9876048..461a9f7 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader.rs @@ -93,7 +93,7 @@ pub trait MyNoSqlDataReader< &'s self, partition_key: &'s str, row_key: &'s str, - ) -> GetEntityBuilder; + ) -> GetEntityBuilder<'s, TMyNoSqlEntity>; async fn has_partition(&self, partition_key: &str) -> bool; diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index ec65c3c..4c490c9 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -104,7 +104,7 @@ where &'s self, partition_key: &'s str, row_key: &'s str, - ) -> GetEntityBuilder { + ) -> GetEntityBuilder<'s, TMyNoSqlEntity> { GetEntityBuilder::new(partition_key, row_key, self.inner.clone()) } @@ -339,7 +339,7 @@ where &'s self, partition_key: &'s str, row_key: &'s str, - ) -> GetEntityBuilder { + ) -> GetEntityBuilder<'s, TMyNoSqlEntity> { self.get_entity_with_callback_to_server(partition_key, row_key) } From 78d83a6c02f413d234f80d7f963a73cd59b7a214 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 24 Dec 2024 00:33:36 +0200 Subject: [PATCH 53/89] Fixes --- .../src/my_no_sql_data_writer/fl_url_factory.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs index 1869681..ab615b3 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/fl_url_factory.rs @@ -42,7 +42,8 @@ impl FlUrlFactory { #[cfg(feature = "with-ssh")] if let Some(ssh_security_credentials_resolver) = &self.ssh_security_credentials_resolver { - return fl_url.set_ssh_private_key_resolver(ssh_security_credentials_resolver.clone()); + return fl_url + .set_ssh_security_credentials_resolver(ssh_security_credentials_resolver.clone()); } fl_url From 96d040c0d1e99920f3ccb0fbda4118c5f712106d Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 24 Dec 2024 12:21:03 +0200 Subject: [PATCH 54/89] Debugging --- .../src/my_no_sql_data_writer/execution.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index db8a834..10690f5 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -99,9 +99,11 @@ pub async fn insert_or_replace_entity< return Ok(()); } - let reason = response.receive_body().await?; - let reason = String::from_utf8(reason)?; - return Err(DataWriterError::Error(reason)); + let body = response.receive_body().await?; + let body = String::from_utf8(body)?; + + println!("Insert or replace response: '{}'", body); + return Err(DataWriterError::Error(body)); } pub async fn bulk_insert_or_replace< From 677918fc119f90f21058c8cdee17c365e29cdd1d Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 24 Dec 2024 12:31:58 +0200 Subject: [PATCH 55/89] Fixes --- my-no-sql-core/src/db/db_row/db_row.rs | 70 ++++++++++----- .../src/db/db_row/test_expires_update.rs | 90 +++++++++++-------- 2 files changed, 98 insertions(+), 62 deletions(-) diff --git a/my-no-sql-core/src/db/db_row/db_row.rs b/my-no-sql-core/src/db/db_row/db_row.rs index dd193e8..1f4a83a 100644 --- a/my-no-sql-core/src/db/db_row/db_row.rs +++ b/my-no-sql-core/src/db/db_row/db_row.rs @@ -114,7 +114,7 @@ impl DbRow { } } #[cfg(feature = "master-node")] - pub fn write_json(&self, out: &mut Vec) { + pub fn write_json(&self, out: &mut String) { let expires_value = self.get_expires(); if expires_value.is_none() { @@ -122,23 +122,39 @@ impl DbRow { if let Some(before_separator) = find_json_separator_before(&self.raw, expires.key.start - 1) { - out.extend_from_slice(&self.raw[..before_separator]); - out.extend_from_slice(&self.raw[expires.value.end..]); + unsafe { + out.push_str(std::str::from_utf8_unchecked(&self.raw[..before_separator])); + out.push_str(std::str::from_utf8_unchecked( + &self.raw[expires.value.end..], + )); + } return; } if let Some(after_separator) = find_json_separator_after(&self.raw, expires.value.end) { - out.extend_from_slice(&self.raw[..expires.key.start]); - out.extend_from_slice(&self.raw[after_separator..]); + unsafe { + out.push_str(std::str::from_utf8_unchecked( + &self.raw[..expires.key.start], + )); + out.push_str(std::str::from_utf8_unchecked(&self.raw[after_separator..])); + } return; } - out.extend_from_slice(&self.raw[..expires.key.start]); - out.extend_from_slice(&self.raw[expires.value.end..]); + unsafe { + out.push_str(std::str::from_utf8_unchecked( + &self.raw[..expires.key.start], + )); + out.push_str(std::str::from_utf8_unchecked( + &self.raw[expires.value.end..], + )); + } } else { - out.extend_from_slice(&self.raw); + unsafe { + out.push_str(std::str::from_utf8_unchecked(&self.raw)); + } } return; @@ -146,16 +162,22 @@ impl DbRow { let expires_value = expires_value.unwrap(); - if let Some(expires) = &self.expires { - out.extend_from_slice(&self.raw[..expires.key.start]); - inject_expires(out, expires_value); - out.extend_from_slice(&self.raw[expires.value.end..]); - } else { - let end_of_json = crate::db_json_entity::get_the_end_of_the_json(&self.raw); - out.extend_from_slice(&self.raw[..end_of_json]); - out.push(b','); - inject_expires(out, expires_value); - out.extend_from_slice(&self.raw[end_of_json..]); + unsafe { + if let Some(expires) = &self.expires { + out.push_str(std::str::from_utf8_unchecked( + &self.raw[..expires.key.start], + )); + inject_expires(out, expires_value); + out.push_str(std::str::from_utf8_unchecked( + &self.raw[expires.value.end..], + )); + } else { + let end_of_json = crate::db_json_entity::get_the_end_of_the_json(&self.raw); + out.push_str(std::str::from_utf8_unchecked(&self.raw[..end_of_json])); + out.push(','); + inject_expires(out, expires_value); + out.push_str(std::str::from_utf8_unchecked(&self.raw[end_of_json..])); + } } } @@ -199,12 +221,12 @@ impl RowKeyParameter for Arc { } #[cfg(feature = "master-node")] -fn inject_expires(out: &mut Vec, expires_value: DateTimeAsMicroseconds) { - out.push(b'"'); - out.extend_from_slice(crate::db_json_entity::consts::EXPIRES.as_bytes()); - out.extend_from_slice("\":\"".as_bytes()); - out.extend_from_slice(&expires_value.to_rfc3339().as_bytes()[..19]); - out.push(b'"'); +fn inject_expires(out: &mut String, expires_value: DateTimeAsMicroseconds) { + out.push('"'); + out.push_str(crate::db_json_entity::consts::EXPIRES); + out.push_str("\":\""); + out.push_str(&expires_value.to_rfc3339()[..19]); + out.push('"'); } #[cfg(feature = "master-node")] fn find_json_separator_before(src: &[u8], pos: usize) -> Option { diff --git a/my-no-sql-core/src/db/db_row/test_expires_update.rs b/my-no-sql-core/src/db/db_row/test_expires_update.rs index 5835833..9b8eb6b 100644 --- a/my-no-sql-core/src/db/db_row/test_expires_update.rs +++ b/my-no-sql-core/src/db/db_row/test_expires_update.rs @@ -22,17 +22,20 @@ mod test { db_row.update_expires(new_expires.into()); - let mut result_json = Vec::new(); + let mut result_json = String::new(); db_row.write_json(&mut result_json); - let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_bytes().into()).unwrap(); - assert_eq!(result_entity.get_partition_key(&result_json), "TestPk"); - assert_eq!(result_entity.get_row_key(&result_json), "TestRk"); + assert_eq!( + result_entity.get_partition_key(result_json.as_bytes()), + "TestPk" + ); + assert_eq!(result_entity.get_row_key(result_json.as_bytes()), "TestRk"); assert_eq!( - result_entity.get_expires(&result_json).unwrap(), + result_entity.get_expires(result_json.as_bytes()).unwrap(), &new_expires.to_rfc3339()[..19] ); } @@ -53,17 +56,20 @@ mod test { db_row.update_expires(new_expires.into()); - let mut result_json = Vec::new(); + let mut result_json = String::new(); db_row.write_json(&mut result_json); - let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_bytes().into()).unwrap(); - assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); - assert_eq!(result_entity.get_row_key(&result_json), "Rk"); + assert_eq!( + result_entity.get_partition_key(result_json.as_bytes()), + "Pk" + ); + assert_eq!(result_entity.get_row_key(result_json.as_bytes()), "Rk"); assert_eq!( - result_entity.get_expires(&result_json).unwrap(), + result_entity.get_expires(result_json.as_bytes()).unwrap(), &new_expires.to_rfc3339()[..19] ); } @@ -80,18 +86,21 @@ mod test { db_row.update_expires(None); - let mut result_json = Vec::new(); + let mut result_json = String::new(); db_row.write_json(&mut result_json); - println!("Result: {}", std::str::from_utf8(&result_json).unwrap()); + println!("Result: {}", result_json); - let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_bytes().into()).unwrap(); - assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); - assert_eq!(result_entity.get_row_key(&result_json), "Rk"); + assert_eq!( + result_entity.get_partition_key(result_json.as_bytes()), + "Pk" + ); + assert_eq!(result_entity.get_row_key(result_json.as_bytes()), "Rk"); - assert!(result_entity.get_expires(&result_json).is_none()); + assert!(result_entity.get_expires(result_json.as_bytes()).is_none()); } #[test] @@ -105,18 +114,21 @@ mod test { db_row.update_expires(None); - let mut result_json = Vec::new(); + let mut result_json = String::new(); db_row.write_json(&mut result_json); - println!("Result: {}", std::str::from_utf8(&result_json).unwrap()); + println!("Result: {}", result_json); - let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_bytes().into()).unwrap(); - assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); - assert_eq!(result_entity.get_row_key(&result_json), "Rk"); + assert_eq!( + result_entity.get_partition_key(result_json.as_bytes()), + "Pk" + ); + assert_eq!(result_entity.get_row_key(result_json.as_bytes()), "Rk"); - assert!(result_entity.get_expires(&result_json).is_none()); + assert!(result_entity.get_expires(result_json.as_bytes()).is_none()); } #[test] @@ -130,18 +142,21 @@ mod test { db_row.update_expires(None); - let mut result_json = Vec::new(); + let mut result_json = String::new(); db_row.write_json(&mut result_json); - println!("Result: {}", std::str::from_utf8(&result_json).unwrap()); + println!("Result: {}", result_json); - let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_bytes().into()).unwrap(); - assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); - assert_eq!(result_entity.get_row_key(&result_json), "Rk"); + assert_eq!( + result_entity.get_partition_key(result_json.as_bytes()), + "Pk" + ); + assert_eq!(result_entity.get_row_key(result_json.as_bytes()), "Rk"); - assert!(result_entity.get_expires(&result_json).is_none()); + assert!(result_entity.get_expires(result_json.as_bytes()).is_none()); } #[test] @@ -158,21 +173,20 @@ mod test { db_row.update_expires(None); - let mut result_json = Vec::new(); + let mut result_json = String::new(); db_row.write_json(&mut result_json); - println!( - "Result: {}. Len: {}", - std::str::from_utf8(&result_json).unwrap(), - result_json.len() - ); + println!("Result: {}. Len: {}", result_json, result_json.len()); - let result_entity = DbJsonEntity::new(result_json.as_slice().into()).unwrap(); + let result_entity = DbJsonEntity::new(result_json.as_bytes().into()).unwrap(); - assert_eq!(result_entity.get_partition_key(&result_json), "Pk"); - assert_eq!(result_entity.get_row_key(&result_json), "Rk"); + assert_eq!( + result_entity.get_partition_key(result_json.as_bytes()), + "Pk" + ); + assert_eq!(result_entity.get_row_key(result_json.as_bytes()), "Rk"); - assert!(result_entity.get_expires(&result_json).is_none()); + assert!(result_entity.get_expires(result_json.as_bytes()).is_none()); } } From 467a6b5a8b68fe7b0670270d58d49420292e7e3b Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 24 Dec 2024 12:38:14 +0200 Subject: [PATCH 56/89] Debugging --- .../src/my_no_sql_data_writer/execution.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 10690f5..437ad47 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -87,15 +87,26 @@ pub async fn insert_or_replace_entity< entity: &TEntity, sync_period: &DataSynchronizationPeriod, ) -> Result<(), DataWriterError> { + let entity = entity.serialize_entity(); + + println!( + "Insert or replace entity: '{}'", + std::str::from_utf8(&entity).unwrap() + ); + let response = flurl .append_path_segment(ROW_CONTROLLER) .append_path_segment("InsertOrReplace") .append_data_sync_period(sync_period) .with_table_name_as_query_param(TEntity::TABLE_NAME) - .post(entity.serialize_entity().into()) + .post(entity.into()) .await?; if is_ok_result(&response) { + let body = response.receive_body().await?; + let body = String::from_utf8(body)?; + + println!("Insert or replace response: '{}'", body); return Ok(()); } From ca15ce6b6f5cb9fe6b0ea24f968885bab1041e8c Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 24 Dec 2024 13:53:14 +0200 Subject: [PATCH 57/89] Removed Debug Data --- .../src/my_no_sql_data_writer/execution.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 437ad47..2b1d04e 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -89,11 +89,6 @@ pub async fn insert_or_replace_entity< ) -> Result<(), DataWriterError> { let entity = entity.serialize_entity(); - println!( - "Insert or replace entity: '{}'", - std::str::from_utf8(&entity).unwrap() - ); - let response = flurl .append_path_segment(ROW_CONTROLLER) .append_path_segment("InsertOrReplace") @@ -103,17 +98,12 @@ pub async fn insert_or_replace_entity< .await?; if is_ok_result(&response) { - let body = response.receive_body().await?; - let body = String::from_utf8(body)?; - - println!("Insert or replace response: '{}'", body); return Ok(()); } let body = response.receive_body().await?; let body = String::from_utf8(body)?; - println!("Insert or replace response: '{}'", body); return Err(DataWriterError::Error(body)); } From c2bec5b95e343776cbf6e45086863d9ef192a4ae Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 26 Dec 2024 22:11:01 +0200 Subject: [PATCH 58/89] Fixes --- my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 2b1d04e..228f6b7 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -509,7 +509,7 @@ async fn check_error(response: &mut FlUrlResponse) -> Result<(), DataWriterError if let Err(err) = &result { my_logger::LOGGER.write_error( - format!("FlUrlRequest to {}", response.url.as_str()), + format!("FlUrlRequest to {}", response.url.to_string()), format!("{:?}", err), None.into(), ); From 855350df4aa12d7a7d4a5f1fe71e9e87a04fb44c Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 13 Feb 2025 11:12:49 +0200 Subject: [PATCH 59/89] Debugging --- .../subscribers/my_no_sql_data_reader_tcp.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 4c490c9..7183642 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -174,11 +174,20 @@ where .into(), ) } else { - LazyMyNoSqlEntity::Deserialized( - TMyNoSqlEntity::deserialize_entity(db_entity_data.as_bytes()) - .unwrap() - .into(), - ) + let result = TMyNoSqlEntity::deserialize_entity(db_entity_data.as_bytes()); + + match result { + Ok(result) => LazyMyNoSqlEntity::Deserialized(Arc::new(result)), + Err(err) => { + println!( + "Invalid entity to deserialize. Table: {}. Content: {:?}", + TMyNoSqlEntity::TABLE_NAME, + db_entity_data.as_str() + ); + + panic!("Can not lazy deserialize entity. Err: {}", err); + } + } }; let partition_key = item_to_insert.get_partition_key(); From 1960379de61ad5ed5bb7553559f558ea1c7268a8 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 13 Feb 2025 16:30:37 +0200 Subject: [PATCH 60/89] Made timestamp Typed --- my-no-sql-abstractions/Cargo.toml | 3 + my-no-sql-abstractions/src/lib.rs | 2 + .../src/my_no_sql_entity.rs | 4 +- my-no-sql-abstractions/src/timestamp_type.rs | 119 ++++++++++++++++++ .../src/my_no_sql_data_writer/execution.rs | 6 +- my-no-sql-macros/src/entity_utils.rs | 8 +- .../src/enum_of_my_no_sql_entity/generate.rs | 2 +- .../src/subscribers/callback_triggers.rs | 46 ++----- .../subscribers/my_no_sql_data_reader_tcp.rs | 30 ----- my-no-sql-tests/src/lib.rs | 2 + my-no-sql-tests/src/macros_tests/enum_test.rs | 4 +- .../test_enum_with_added_new_type.rs | 2 +- my-no-sql-tests/src/test_same_timestamp.rs | 9 ++ 13 files changed, 161 insertions(+), 76 deletions(-) create mode 100644 my-no-sql-abstractions/src/timestamp_type.rs create mode 100644 my-no-sql-tests/src/test_same_timestamp.rs diff --git a/my-no-sql-abstractions/Cargo.toml b/my-no-sql-abstractions/Cargo.toml index 8db7846..c6319a2 100644 --- a/my-no-sql-abstractions/Cargo.toml +++ b/my-no-sql-abstractions/Cargo.toml @@ -6,3 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } +serde = { version = "*", features = ["derive"] } +serde_json = "*" diff --git a/my-no-sql-abstractions/src/lib.rs b/my-no-sql-abstractions/src/lib.rs index 72dce9b..2618c19 100644 --- a/my-no-sql-abstractions/src/lib.rs +++ b/my-no-sql-abstractions/src/lib.rs @@ -2,3 +2,5 @@ mod data_sync_period; mod my_no_sql_entity; pub use data_sync_period::DataSynchronizationPeriod; pub use my_no_sql_entity::*; +mod timestamp_type; +pub use timestamp_type::*; diff --git a/my-no-sql-abstractions/src/my_no_sql_entity.rs b/my-no-sql-abstractions/src/my_no_sql_entity.rs index 200f661..98b886a 100644 --- a/my-no-sql-abstractions/src/my_no_sql_entity.rs +++ b/my-no-sql-abstractions/src/my_no_sql_entity.rs @@ -1,9 +1,11 @@ +use crate::Timestamp; + pub trait MyNoSqlEntity { const TABLE_NAME: &'static str; const LAZY_DESERIALIZATION: bool; fn get_partition_key(&self) -> &str; fn get_row_key(&self) -> &str; - fn get_time_stamp(&self) -> i64; + fn get_time_stamp(&self) -> Timestamp; } pub trait MyNoSqlEntitySerializer: Sized { diff --git a/my-no-sql-abstractions/src/timestamp_type.rs b/my-no-sql-abstractions/src/timestamp_type.rs new file mode 100644 index 0000000..4d084cd --- /dev/null +++ b/my-no-sql-abstractions/src/timestamp_type.rs @@ -0,0 +1,119 @@ +use std::fmt::{Debug, Display}; + +use rust_extensions::date_time::DateTimeAsMicroseconds; +use serde::{Deserialize, Deserializer}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Timestamp(i64); + +impl Timestamp { + pub fn to_date_time(&self) -> DateTimeAsMicroseconds { + DateTimeAsMicroseconds::new(self.0) + } +} + +impl Into for DateTimeAsMicroseconds { + fn into(self) -> Timestamp { + Timestamp(self.unix_microseconds) + } +} + +impl Into for Timestamp { + fn into(self) -> DateTimeAsMicroseconds { + DateTimeAsMicroseconds::new(self.0) + } +} + +impl Display for Timestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let timestamp = self.to_date_time().to_rfc3339(); + f.write_str(timestamp.as_str()) + } +} + +impl Debug for Timestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let timestamp = self.to_date_time().to_rfc3339(); + f.debug_tuple("Timestamp").field(×tamp).finish() + } +} + +impl serde::Serialize for Timestamp { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let rfc3339 = self.to_date_time().to_rfc3339(); + serializer.serialize_str(&rfc3339) + } +} + +impl<'de> Deserialize<'de> for Timestamp { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let datetime = DateTimeAsMicroseconds::from_str(s.as_str()); + + if datetime.is_none() { + println!("Can not parse timestamp: {}", s); + } + Ok(Timestamp(datetime.unwrap().unix_microseconds)) + } +} + +impl Default for Timestamp { + fn default() -> Self { + Self(0) + } +} + +impl Into for i64 { + fn into(self) -> Timestamp { + Timestamp(self) + } +} + +impl Into for u64 { + fn into(self) -> Timestamp { + Timestamp(self as i64) + } +} + +#[cfg(test)] +mod test { + use rust_extensions::date_time::DateTimeAsMicroseconds; + use serde::{Deserialize, Serialize}; + + use super::Timestamp; + + #[derive(Debug, Serialize, Deserialize)] + pub struct MyType { + pub my_field: i32, + pub timestamp: Timestamp, + } + + #[test] + fn test_serialization() { + use rust_extensions::date_time::DateTimeAsMicroseconds; + + let my_type = MyType { + my_field: 15, + timestamp: DateTimeAsMicroseconds::from_str("2025-01-01T12:00:00.123456") + .unwrap() + .into(), + }; + + println!("{:?}", my_type); + + let serialized = serde_json::to_string(&my_type).unwrap(); + + println!("Serialized: {}", serialized); + + let result_type: MyType = serde_json::from_str(serialized.as_str()).unwrap(); + + assert_eq!(my_type.my_field, result_type.my_field); + assert_eq!(my_type.timestamp.0, result_type.timestamp.0); + } +} diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 228f6b7..9153197 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -607,7 +607,7 @@ async fn create_table_errors_handler( #[cfg(test)] mod tests { - use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; + use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer, Timestamp}; use serde::Serialize; use serde_derive::Deserialize; @@ -630,8 +630,8 @@ mod tests { &self.row_key } - fn get_time_stamp(&self) -> i64 { - 0 + fn get_time_stamp(&self) -> Timestamp { + Timestamp::default() } } diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index 471c35c..58a58e1 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -89,16 +89,14 @@ pub fn get_row_key_token() -> proc_macro2::TokenStream { pub fn get_time_stamp_token() -> proc_macro2::TokenStream { quote::quote! { #[serde(rename = "TimeStamp")] - pub time_stamp: String, + pub time_stamp: my_no_sql_sdk::abstractions::Timestamp, } } pub fn get_fn_get_time_stamp_token() -> proc_macro2::TokenStream { quote::quote! { - fn get_time_stamp(&self) -> i64 { - my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds::parse_iso_string(self.time_stamp.as_str()) - .unwrap() - .unix_microseconds + fn get_time_stamp(&self) -> my_no_sql_sdk::abstractions::Timestamp { + self.time_stamp } } } diff --git a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs index f67361a..ea4077a 100644 --- a/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/enum_of_my_no_sql_entity/generate.rs @@ -66,7 +66,7 @@ pub fn generate( } } - fn get_time_stamp(&self) -> i64 { + fn get_time_stamp(&self) -> my_no_sql_sdk::abstractions::Timestamp { use my_no_sql_sdk::abstractions::MyNoSqlEntity; match self { #time_stamps diff --git a/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs b/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs index 766ee3a..d3ba745 100644 --- a/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs +++ b/my-no-sql-tcp-reader/src/subscribers/callback_triggers.rs @@ -9,13 +9,8 @@ pub async fn trigger_table_difference< TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, - before: Option< - BTreeMap>>, - >, - now_entities: &BTreeMap< - String, - BTreeMap>, - >, + before: Option>>>, + now_entities: &BTreeMap>>, ) { match before { Some(before) => { @@ -32,10 +27,7 @@ pub async fn trigger_brand_new_table< TMyNoSqlDataReaderCallBacks: MyNoSqlDataReaderCallBacks, >( callbacks: &TMyNoSqlDataReaderCallBacks, - now_entities: &BTreeMap< - String, - BTreeMap>, - >, + now_entities: &BTreeMap>>, ) { for (partition_key, now_partition) in now_entities { let mut added_entities = Vec::new(); @@ -57,10 +49,7 @@ pub async fn trigger_old_and_new_table_difference< >( callbacks: &TMyNoSqlDataReaderCallBacks, mut before: BTreeMap>>, - now_entities: &BTreeMap< - String, - BTreeMap>, - >, + now_entities: &BTreeMap>>, ) { for (now_partition_key, now_partition) in now_entities { let before_partition = before.remove(now_partition_key); @@ -159,15 +148,14 @@ pub async fn trigger_brand_new_partition< mod tests { use std::collections::BTreeMap; - use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer}; + use my_no_sql_abstractions::{MyNoSqlEntity, MyNoSqlEntitySerializer, Timestamp}; use serde_derive::{Deserialize, Serialize}; use tokio::sync::Mutex; use crate::subscribers::{LazyMyNoSqlEntity, MyNoSqlDataReaderCallBacks}; struct TestCallbacksInner { - inserted_or_replaced_entities: - BTreeMap>>, + inserted_or_replaced_entities: BTreeMap>>, deleted: BTreeMap>>, } @@ -210,11 +198,7 @@ mod tests { } } - async fn deleted( - &self, - partition_key: &str, - entities: Vec>, - ) { + async fn deleted(&self, partition_key: &str, entities: Vec>) { let mut write_access = self.data.lock().await; match write_access.deleted.get_mut(partition_key) { Some(db_partition) => { @@ -257,8 +241,8 @@ mod tests { fn get_row_key(&self) -> &str { self.row_key.as_str() } - fn get_time_stamp(&self) -> i64 { - self.timestamp + fn get_time_stamp(&self) -> Timestamp { + self.timestamp.into() } } @@ -276,8 +260,7 @@ mod tests { pub async fn test_we_had_data_in_table_and_new_table_is_empty() { let test_callback = TestCallbacks::new(); - let mut before_rows: BTreeMap> = - BTreeMap::new(); + let mut before_rows: BTreeMap> = BTreeMap::new(); before_rows.insert( "RK1".to_string(), @@ -305,8 +288,7 @@ mod tests { pub async fn test_brand_new_table() { let test_callback = TestCallbacks::new(); - let mut after_rows: BTreeMap> = - BTreeMap::new(); + let mut after_rows: BTreeMap> = BTreeMap::new(); after_rows.insert( "RK1".to_string(), @@ -338,8 +320,7 @@ mod tests { pub async fn test_we_have_updates_in_table() { let test_callback = TestCallbacks::new(); - let mut before_partition: BTreeMap> = - BTreeMap::new(); + let mut before_partition: BTreeMap> = BTreeMap::new(); before_partition.insert( "RK1".to_string(), @@ -353,8 +334,7 @@ mod tests { let mut before = BTreeMap::new(); before.insert("PK1".to_string(), before_partition); - let mut after_partition: BTreeMap> = - BTreeMap::new(); + let mut after_partition: BTreeMap> = BTreeMap::new(); after_partition.insert( "RK2".to_string(), TestRow::new("PK1".to_string(), "RK2".to_string(), 2).into(), diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs index 7183642..10d892e 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_tcp.rs @@ -199,36 +199,6 @@ where } result - - /* - - let mut result = BTreeMap::new(); - - for db_entity in JsonArrayIterator::new(data) { - if let Err(err) = &db_entity { - panic!( - "Table: {}. The whole array of json entities is broken. Err: {:?}", - TMyNoSqlEntity::TABLE_NAME, - err - ); - } - - let db_entity_data = db_entity.unwrap(); - - let el = TMyNoSqlEntity::deserialize_entity(db_entity_data); - - if let Some(el) = el { - let partition_key = el.get_partition_key(); - if !result.contains_key(partition_key) { - result.insert(partition_key.to_string(), Vec::new()); - } - - result.get_mut(partition_key).unwrap().push(el); - } - } - - result - */ } pub async fn get_enum_case_models_by_partition_key< diff --git a/my-no-sql-tests/src/lib.rs b/my-no-sql-tests/src/lib.rs index cd886f6..6be0a80 100644 --- a/my-no-sql-tests/src/lib.rs +++ b/my-no-sql-tests/src/lib.rs @@ -2,3 +2,5 @@ mod macros_tests; #[cfg(test)] mod test_new_enum_cases_added; +#[cfg(test)] +mod test_same_timestamp; diff --git a/my-no-sql-tests/src/macros_tests/enum_test.rs b/my-no-sql-tests/src/macros_tests/enum_test.rs index 8f268e5..3810336 100644 --- a/my-no-sql-tests/src/macros_tests/enum_test.rs +++ b/my-no-sql-tests/src/macros_tests/enum_test.rs @@ -25,7 +25,7 @@ pub struct Struct2 { #[test] fn test_serialize_deserialize_case_1() { let src_model = Struct1 { - time_stamp: "".to_string(), + time_stamp: Default::default(), field1: "test".to_string(), field2: 123, }; @@ -45,7 +45,7 @@ fn test_serialize_deserialize_case_1() { fn test_serialize_deserialize_case_2() { let src_model = Struct2 { row_key: "rk2".to_string(), - time_stamp: "".to_string(), + time_stamp: Default::default(), field3: "test3".to_string(), field4: 1234, }; diff --git a/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs b/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs index 0663b46..c34f96f 100644 --- a/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs +++ b/my-no-sql-tests/src/test_new_enum_cases_added/test_enum_with_added_new_type.rs @@ -37,7 +37,7 @@ mod tests { #[test] fn test() { let model_ver2 = MyNoSqlEnumEntityTestVer2::Case2(Struct2 { - time_stamp: "".to_string(), + time_stamp: Default::default(), field3: "field3".to_string(), field4: 4, }); diff --git a/my-no-sql-tests/src/test_same_timestamp.rs b/my-no-sql-tests/src/test_same_timestamp.rs new file mode 100644 index 0000000..cf80e0a --- /dev/null +++ b/my-no-sql-tests/src/test_same_timestamp.rs @@ -0,0 +1,9 @@ +use my_no_sql_macros::my_no_sql_entity; +use my_no_sql_sdk::abstractions::Timestamp; +use serde::{Deserialize, Serialize}; + +#[my_no_sql_entity("test-table")] +#[derive(Debug, Serialize, Deserialize)] +pub struct MyEntity { + pub expires: Timestamp, +} From d604a9061018eb5d8bcd6ad741a22f612b8b49bc Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 13 Feb 2025 20:26:27 +0200 Subject: [PATCH 61/89] Timestamp is now has types --- my-no-sql-abstractions/src/timestamp_type.rs | 65 ++++++++++++-- my-no-sql-macros/src/entity_utils.rs | 89 ++++++++++++++++++- my-no-sql-macros/src/enum_model/generate.rs | 2 +- my-no-sql-macros/src/lib.rs | 2 +- .../src/my_no_sql_entity/generate.rs | 18 +++- .../my_no_sql_entity/generate_base_impl.rs | 9 +- my-no-sql-tests/src/test_same_timestamp.rs | 37 +++++++- 7 files changed, 205 insertions(+), 17 deletions(-) diff --git a/my-no-sql-abstractions/src/timestamp_type.rs b/my-no-sql-abstractions/src/timestamp_type.rs index 4d084cd..def28da 100644 --- a/my-no-sql-abstractions/src/timestamp_type.rs +++ b/my-no-sql-abstractions/src/timestamp_type.rs @@ -10,6 +10,13 @@ impl Timestamp { pub fn to_date_time(&self) -> DateTimeAsMicroseconds { DateTimeAsMicroseconds::new(self.0) } + pub fn is_default(&self) -> bool { + self.0 == 0 + } + + pub fn to_i64(&self) -> i64 { + self.0 + } } impl Into for DateTimeAsMicroseconds { @@ -26,6 +33,10 @@ impl Into for Timestamp { impl Display for Timestamp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.0 == 0 { + return f.write_str("null"); + } + let timestamp = self.to_date_time().to_rfc3339(); f.write_str(timestamp.as_str()) } @@ -33,8 +44,12 @@ impl Display for Timestamp { impl Debug for Timestamp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let timestamp = self.to_date_time().to_rfc3339(); - f.debug_tuple("Timestamp").field(×tamp).finish() + if self.0 == 0 { + f.debug_tuple("Timestamp").field(&"null").finish() + } else { + let timestamp = self.to_date_time().to_rfc3339(); + f.debug_tuple("Timestamp").field(×tamp).finish() + } } } @@ -43,8 +58,15 @@ impl serde::Serialize for Timestamp { where S: serde::Serializer, { + if self.0 == 0 { + return serializer.serialize_none(); + } let rfc3339 = self.to_date_time().to_rfc3339(); - serializer.serialize_str(&rfc3339) + + match rfc3339.find("+") { + Some(index) => serializer.serialize_str(&rfc3339[..index]), + None => serializer.serialize_str(&rfc3339), + } } } @@ -53,12 +75,20 @@ impl<'de> Deserialize<'de> for Timestamp { where D: Deserializer<'de>, { - let s = String::deserialize(deserializer)?; + let s = String::deserialize(deserializer); + + if s.is_err() { + return Ok(Timestamp(0)); + } + + let s = s.unwrap(); + let datetime = DateTimeAsMicroseconds::from_str(s.as_str()); if datetime.is_none() { println!("Can not parse timestamp: {}", s); } + Ok(Timestamp(datetime.unwrap().unix_microseconds)) } } @@ -81,9 +111,12 @@ impl Into for u64 { } } +pub fn skip_timestamp_serializing(timestamp: &Timestamp) -> bool { + timestamp.is_default() +} + #[cfg(test)] mod test { - use rust_extensions::date_time::DateTimeAsMicroseconds; use serde::{Deserialize, Serialize}; use super::Timestamp; @@ -91,6 +124,7 @@ mod test { #[derive(Debug, Serialize, Deserialize)] pub struct MyType { pub my_field: i32, + #[serde(skip_serializing_if = "super::skip_timestamp_serializing")] pub timestamp: Timestamp, } @@ -116,4 +150,25 @@ mod test { assert_eq!(my_type.my_field, result_type.my_field); assert_eq!(my_type.timestamp.0, result_type.timestamp.0); } + + #[test] + fn test_serialization_none() { + use rust_extensions::date_time::DateTimeAsMicroseconds; + + let my_type = MyType { + my_field: 15, + timestamp: DateTimeAsMicroseconds::new(0).into(), + }; + + println!("{:?}", my_type); + + let serialized = serde_json::to_string(&my_type).unwrap(); + + println!("Serialized: {}", serialized); + + let result_type: MyType = serde_json::from_str(serialized.as_str()).unwrap(); + + assert_eq!(my_type.my_field, result_type.my_field); + assert_eq!(my_type.timestamp.0, result_type.timestamp.0); + } } diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index 58a58e1..3efef44 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -1,11 +1,15 @@ -use quote::ToTokens; +use std::collections::HashSet; -pub fn compile_struct_with_new_fields( +use quote::{quote, ToTokens}; +use syn::Ident; +use types_reader::StructProperty; + +pub fn compile_src_with_new_fields( ast: &proc_macro2::TokenStream, add_pk: bool, add_rk: bool, add_timestamp: bool, -) -> (syn::Ident, proc_macro2::TokenStream) { +) -> (Ident, proc_macro2::TokenStream) { let mut result: Vec = Vec::new(); let mut struct_name = None; let mut passed_struct_name = false; @@ -54,6 +58,85 @@ pub fn compile_struct_with_new_fields( (struct_name, quote::quote!(#(#result)*)) } +pub fn extract_derive(ast: &proc_macro2::TokenStream) -> proc_macro2::TokenStream { + let mut derive_result = Vec::new(); + + for item in ast.into_token_stream() { + let last_token = if let proc_macro2::TokenTree::Group(_) = &item { + true + } else { + false + }; + derive_result.push(item); + + if last_token { + break; + } + } + + quote! {#(#derive_result)*} +} + +pub fn compile_struct_with_new_fields( + struct_name: &Ident, + derive: proc_macro2::TokenStream, + fields: &[StructProperty], +) -> Result { + let mut structure_fields = Vec::new(); + + let mut serde_fields = HashSet::new(); + serde_fields.insert("PartitionKey"); + serde_fields.insert("RowKey"); + serde_fields.insert("TimeStamp"); + + for field in fields { + if field.name == "expires" { + if !field.ty.as_str().as_str().ends_with("Timestamp") { + return field.throw_error("Field must be a Timestamp"); + } + } + + if let Some(rename_attr) = field.attrs.try_get_attr("serde") { + let field_name = rename_attr.get_named_param("rename")?; + let field_name = field_name.unwrap_any_value_as_str()?; + + let field_name = field_name.as_str()?; + if serde_fields.contains(field_name) { + return field.throw_error("Field with the same Serde name exists"); + } + + serde_fields.insert(field_name); + } + + if serde_fields.contains(field.name.as_str()) { + return field.throw_error("Field with the same Serde name exists"); + } + + serde_fields.insert(field.name.as_str()); + + let field = field.field; + + structure_fields.push(quote::quote! {#field,}); + } + // #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] + let result = quote! { + + #derive + pub struct #struct_name{ + #[serde(rename="PartitionKey")] + pub partition_key: String, + #[serde(rename="RowKey")] + pub row_key: String, + #[serde(rename="TimeStamp")] + #[serde(skip_serializing_if = "my_no_sql_sdk::abstractions::skip_timestamp_serializing")] + pub time_stamp: my_no_sql_sdk::abstractions::Timestamp, + #(#structure_fields)* + } + }; + + Ok(result) +} + fn populate_tokens( result_tokens: &mut Vec, add_pk: bool, diff --git a/my-no-sql-macros/src/enum_model/generate.rs b/my-no-sql-macros/src/enum_model/generate.rs index a75f367..d73e8a7 100644 --- a/my-no-sql-macros/src/enum_model/generate.rs +++ b/my-no-sql-macros/src/enum_model/generate.rs @@ -14,7 +14,7 @@ pub fn generate( let row_key = parameters.row_key; let (struct_name, new_struct) = - compile_struct_with_new_fields(&input, false, row_key.is_none(), true); + compile_src_with_new_fields(&input, false, row_key.is_none(), true); let fn_get_time_stamp = get_fn_get_time_stamp_token(); diff --git a/my-no-sql-macros/src/lib.rs b/my-no-sql-macros/src/lib.rs index fbe9417..7dd56d3 100644 --- a/my-no-sql-macros/src/lib.rs +++ b/my-no-sql-macros/src/lib.rs @@ -15,7 +15,7 @@ struct MyNoSqlEntityParameters<'s> { #[proc_macro_attribute] pub fn my_no_sql_entity(attr: TokenStream, input: TokenStream) -> TokenStream { - match crate::my_no_sql_entity::generate(attr.into(), input.into()) { + match crate::my_no_sql_entity::generate(attr.into(), input) { Ok(result) => result.into(), Err(err) => err.into_compile_error().into(), } diff --git a/my-no-sql-macros/src/my_no_sql_entity/generate.rs b/my-no-sql-macros/src/my_no_sql_entity/generate.rs index 3a19e14..b2e72d6 100644 --- a/my-no-sql-macros/src/my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/my_no_sql_entity/generate.rs @@ -1,21 +1,31 @@ extern crate proc_macro; use proc_macro::TokenStream; -use types_reader::TokensObject; +use syn::DeriveInput; +use types_reader::{StructProperty, TokensObject}; use crate::MyNoSqlEntityParameters; pub fn generate( attr: proc_macro2::TokenStream, - input: proc_macro2::TokenStream, + input: proc_macro::TokenStream, ) -> Result { - let ast = proc_macro2::TokenStream::from(input); + let input_token_stream: proc_macro2::TokenStream = input.clone().into(); + + let derive = crate::entity_utils::extract_derive(&input_token_stream); + + let input: DeriveInput = syn::parse(input).unwrap(); + + let struct_name = &input.ident; + + let fields = StructProperty::read(&input)?; let attr: TokensObject = attr.try_into()?; let params = MyNoSqlEntityParameters::try_from(&attr)?; - let result = super::generate_base_impl(&ast, params.table_name)?; + let result = + super::generate_base_impl(struct_name, derive, fields.as_slice(), params.table_name)?; Ok(result.into()) } diff --git a/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs b/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs index 4886faa..1e50491 100644 --- a/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs +++ b/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs @@ -1,10 +1,15 @@ +use syn::Ident; +use types_reader::StructProperty; + use crate::entity_utils::*; pub fn generate_base_impl( - ast: &proc_macro2::TokenStream, + struct_name: &Ident, + derive: proc_macro2::TokenStream, + fields: &[StructProperty], table_name: &str, ) -> Result { - let (struct_name, new_struct) = compile_struct_with_new_fields(ast, true, true, true); + let new_struct = compile_struct_with_new_fields(struct_name, derive, fields)?; let fn_get_time_stamp = get_fn_get_time_stamp_token(); diff --git a/my-no-sql-tests/src/test_same_timestamp.rs b/my-no-sql-tests/src/test_same_timestamp.rs index cf80e0a..78c8ca5 100644 --- a/my-no-sql-tests/src/test_same_timestamp.rs +++ b/my-no-sql-tests/src/test_same_timestamp.rs @@ -1,9 +1,44 @@ use my_no_sql_macros::my_no_sql_entity; use my_no_sql_sdk::abstractions::Timestamp; -use serde::{Deserialize, Serialize}; +use serde::*; #[my_no_sql_entity("test-table")] #[derive(Debug, Serialize, Deserialize)] pub struct MyEntity { pub expires: Timestamp, + pub ts: String, +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use my_no_sql_sdk::{ + abstractions::MyNoSqlEntitySerializer, + core::rust_extensions::date_time::DateTimeAsMicroseconds, + }; + + use super::MyEntity; + + #[test] + fn test() { + let entity = MyEntity { + partition_key: "test".to_string(), + row_key: "test".to_string(), + time_stamp: Default::default(), + expires: DateTimeAsMicroseconds::now() + .add(Duration::from_secs(5)) + .into(), + ts: "str".to_string(), + }; + + let result = entity.serialize_entity(); + + let result = MyEntity::deserialize_entity(&result).unwrap(); + + assert_eq!(entity.partition_key.as_str(), result.partition_key.as_str()); + assert_eq!(entity.row_key.as_str(), result.row_key.as_str()); + assert_eq!(entity.time_stamp, result.time_stamp); + assert_eq!(entity.ts, result.ts); + } } From e0b758c74d91caa6ee76b40e08317736a733bb91 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 13 Feb 2025 20:42:33 +0200 Subject: [PATCH 62/89] Fixing with serder_rename --- my-no-sql-macros/src/entity_utils.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index 3efef44..74f2c16 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -97,15 +97,17 @@ pub fn compile_struct_with_new_fields( } if let Some(rename_attr) = field.attrs.try_get_attr("serde") { - let field_name = rename_attr.get_named_param("rename")?; - let field_name = field_name.unwrap_any_value_as_str()?; + let param_rename = rename_attr.try_get_named_param("rename"); + if let Some(param_rename) = param_rename { + let param_rename = param_rename.unwrap_any_value_as_str()?; + let param_rename = param_rename.as_str()?; - let field_name = field_name.as_str()?; - if serde_fields.contains(field_name) { - return field.throw_error("Field with the same Serde name exists"); - } + if serde_fields.contains(param_rename) { + return field.throw_error("Field with the same Serde name exists"); + } - serde_fields.insert(field_name); + serde_fields.insert(param_rename); + } } if serde_fields.contains(field.name.as_str()) { From eda3c2378b3d1ffda160fc975ddb413b4074c71b Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 13 Feb 2025 20:56:06 +0200 Subject: [PATCH 63/89] Added with_expires:true --- my-no-sql-macros/src/entity_utils.rs | 9 +++++++++ my-no-sql-macros/src/lib.rs | 1 + my-no-sql-macros/src/my_no_sql_entity/generate.rs | 9 +++++++-- .../src/my_no_sql_entity/generate_base_impl.rs | 3 ++- my-no-sql-tests/src/test_same_timestamp.rs | 4 +--- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index 74f2c16..88c8b5d 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -81,6 +81,7 @@ pub fn compile_struct_with_new_fields( struct_name: &Ident, derive: proc_macro2::TokenStream, fields: &[StructProperty], + render_expires: bool, ) -> Result { let mut structure_fields = Vec::new(); @@ -120,6 +121,14 @@ pub fn compile_struct_with_new_fields( structure_fields.push(quote::quote! {#field,}); } + + if render_expires { + structure_fields.push(quote::quote! { + #[serde(rename="Expires")] + pub expires: my_no_sql_sdk::abstractions::Timestamp, + }); + } + // #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] let result = quote! { diff --git a/my-no-sql-macros/src/lib.rs b/my-no-sql-macros/src/lib.rs index 7dd56d3..068ae67 100644 --- a/my-no-sql-macros/src/lib.rs +++ b/my-no-sql-macros/src/lib.rs @@ -11,6 +11,7 @@ mod my_no_sql_entity; struct MyNoSqlEntityParameters<'s> { #[default] pub table_name: &'s str, + pub with_expires: Option, } #[proc_macro_attribute] diff --git a/my-no-sql-macros/src/my_no_sql_entity/generate.rs b/my-no-sql-macros/src/my_no_sql_entity/generate.rs index b2e72d6..74aaa65 100644 --- a/my-no-sql-macros/src/my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/my_no_sql_entity/generate.rs @@ -24,8 +24,13 @@ pub fn generate( let params = MyNoSqlEntityParameters::try_from(&attr)?; - let result = - super::generate_base_impl(struct_name, derive, fields.as_slice(), params.table_name)?; + let result = super::generate_base_impl( + struct_name, + derive, + fields.as_slice(), + params.table_name, + params.with_expires.unwrap_or(false), + )?; Ok(result.into()) } diff --git a/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs b/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs index 1e50491..d5be979 100644 --- a/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs +++ b/my-no-sql-macros/src/my_no_sql_entity/generate_base_impl.rs @@ -8,8 +8,9 @@ pub fn generate_base_impl( derive: proc_macro2::TokenStream, fields: &[StructProperty], table_name: &str, + render_expires: bool, ) -> Result { - let new_struct = compile_struct_with_new_fields(struct_name, derive, fields)?; + let new_struct = compile_struct_with_new_fields(struct_name, derive, fields, render_expires)?; let fn_get_time_stamp = get_fn_get_time_stamp_token(); diff --git a/my-no-sql-tests/src/test_same_timestamp.rs b/my-no-sql-tests/src/test_same_timestamp.rs index 78c8ca5..e61af0b 100644 --- a/my-no-sql-tests/src/test_same_timestamp.rs +++ b/my-no-sql-tests/src/test_same_timestamp.rs @@ -1,11 +1,9 @@ use my_no_sql_macros::my_no_sql_entity; -use my_no_sql_sdk::abstractions::Timestamp; use serde::*; -#[my_no_sql_entity("test-table")] +#[my_no_sql_entity(table_name:"test-table", with_expires:true)] #[derive(Debug, Serialize, Deserialize)] pub struct MyEntity { - pub expires: Timestamp, pub ts: String, } From b83b09d0c31a35baae065f01bd8553d3a4ff5fa9 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 14 Feb 2025 10:53:57 +0200 Subject: [PATCH 64/89] Made deserialization error way understandable --- .../src/my_no_sql_data_writer/execution.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 9153197..51ed0f3 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -567,7 +567,15 @@ fn deserialize_entities( while let Some(item) = json_array_iterator.get_next() { let itm = item.unwrap(); - result.push(TEntity::deserialize_entity(itm.as_bytes()).unwrap()); + match TEntity::deserialize_entity(itm.as_bytes()) { + Ok(entity) => { + result.push(entity); + } + Err(err) => { + println!("Entity: {:?}", std::str::from_utf8(itm.as_bytes())); + panic!("Can not deserialize entity: {}", err); + } + } } Ok(result) From cfb86e543233a30e814c1a87d791f36f5703da7c Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 14 Feb 2025 19:49:08 +0200 Subject: [PATCH 65/89] Added error on table --- .../src/my_no_sql_data_writer/execution.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 51ed0f3..676b7ed 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -572,7 +572,11 @@ fn deserialize_entities( result.push(entity); } Err(err) => { - println!("Entity: {:?}", std::str::from_utf8(itm.as_bytes())); + println!( + "Table: '{}', Entity: {:?}", + TEntity::TABLE_NAME, + std::str::from_utf8(itm.as_bytes()) + ); panic!("Can not deserialize entity: {}", err); } } From a4d8aab72d25b26aee7a90c8266ca10a674d9fab Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 15 Feb 2025 11:31:42 +0200 Subject: [PATCH 66/89] Fixes with #[serde(rename_all = "PascalCase")] --- my-no-sql-abstractions/src/timestamp_type.rs | 17 +++++++++++ my-no-sql-macros/src/entity_utils.rs | 18 +++++------- .../src/my_no_sql_entity/generate.rs | 2 +- my-no-sql-tests/src/lib.rs | 2 ++ my-no-sql-tests/src/tests_from_real_life.rs | 29 +++++++++++++++++++ 5 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 my-no-sql-tests/src/tests_from_real_life.rs diff --git a/my-no-sql-abstractions/src/timestamp_type.rs b/my-no-sql-abstractions/src/timestamp_type.rs index def28da..c0a5e56 100644 --- a/my-no-sql-abstractions/src/timestamp_type.rs +++ b/my-no-sql-abstractions/src/timestamp_type.rs @@ -117,6 +117,7 @@ pub fn skip_timestamp_serializing(timestamp: &Timestamp) -> bool { #[cfg(test)] mod test { + use rust_extensions::date_time::{DateTimeAsMicroseconds, DateTimeStruct}; use serde::{Deserialize, Serialize}; use super::Timestamp; @@ -171,4 +172,20 @@ mod test { assert_eq!(my_type.my_field, result_type.my_field); assert_eq!(my_type.timestamp.0, result_type.timestamp.0); } + + #[test] + fn test_from_real_example() { + let time_stamp = DateTimeAsMicroseconds::from_str("2024-11-29T14:59:15.6145").unwrap(); + + let dt_struct: DateTimeStruct = time_stamp.into(); + + assert_eq!(dt_struct.year, 2024); + assert_eq!(dt_struct.month, 11); + assert_eq!(dt_struct.day, 29); + + assert_eq!(dt_struct.time.hour, 14); + assert_eq!(dt_struct.time.min, 59); + assert_eq!(dt_struct.time.sec, 15); + assert_eq!(dt_struct.time.micros, 614500); + } } diff --git a/my-no-sql-macros/src/entity_utils.rs b/my-no-sql-macros/src/entity_utils.rs index 88c8b5d..c51300c 100644 --- a/my-no-sql-macros/src/entity_utils.rs +++ b/my-no-sql-macros/src/entity_utils.rs @@ -58,20 +58,18 @@ pub fn compile_src_with_new_fields( (struct_name, quote::quote!(#(#result)*)) } -pub fn extract_derive(ast: &proc_macro2::TokenStream) -> proc_macro2::TokenStream { +pub fn extract_attributes(ast: &proc_macro2::TokenStream) -> proc_macro2::TokenStream { let mut derive_result = Vec::new(); for item in ast.into_token_stream() { - let last_token = if let proc_macro2::TokenTree::Group(_) = &item { - true - } else { - false - }; - derive_result.push(item); - - if last_token { - break; + if let proc_macro2::TokenTree::Ident(item) = &item { + let to_str = item.to_string(); + if to_str == "pub" || to_str == "struct" { + break; + } } + + derive_result.push(item); } quote! {#(#derive_result)*} diff --git a/my-no-sql-macros/src/my_no_sql_entity/generate.rs b/my-no-sql-macros/src/my_no_sql_entity/generate.rs index 74aaa65..ebea872 100644 --- a/my-no-sql-macros/src/my_no_sql_entity/generate.rs +++ b/my-no-sql-macros/src/my_no_sql_entity/generate.rs @@ -12,7 +12,7 @@ pub fn generate( ) -> Result { let input_token_stream: proc_macro2::TokenStream = input.clone().into(); - let derive = crate::entity_utils::extract_derive(&input_token_stream); + let derive = crate::entity_utils::extract_attributes(&input_token_stream); let input: DeriveInput = syn::parse(input).unwrap(); diff --git a/my-no-sql-tests/src/lib.rs b/my-no-sql-tests/src/lib.rs index 6be0a80..ab36453 100644 --- a/my-no-sql-tests/src/lib.rs +++ b/my-no-sql-tests/src/lib.rs @@ -4,3 +4,5 @@ mod macros_tests; mod test_new_enum_cases_added; #[cfg(test)] mod test_same_timestamp; +#[cfg(test)] +mod tests_from_real_life; diff --git a/my-no-sql-tests/src/tests_from_real_life.rs b/my-no-sql-tests/src/tests_from_real_life.rs new file mode 100644 index 0000000..1019529 --- /dev/null +++ b/my-no-sql-tests/src/tests_from_real_life.rs @@ -0,0 +1,29 @@ +use my_no_sql_macros::my_no_sql_entity; +use serde::*; + +#[my_no_sql_entity("payout-withdrawal-settings")] +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct PayoutWithdrawalSettingsMyNoSqlEntity { + pub value: f64, + pub currency: String, +} + +#[cfg(test)] +mod tests { + + use my_no_sql_sdk::abstractions::MyNoSqlEntitySerializer; + + use super::PayoutWithdrawalSettingsMyNoSqlEntity; + + #[test] + fn test() { + let src = "{\"PartitionKey\":\"bank-transfer\",\"RowKey\":\"max\",\"TimeStamp\":\"2024-11-29T14:59:15.6145\",\"Value\":15000.0,\"Currency\":\"USD\"}"; + + let entity = + PayoutWithdrawalSettingsMyNoSqlEntity::deserialize_entity(src.as_bytes()).unwrap(); + + assert_eq!(entity.value, 15000.0); + assert_eq!(entity.currency, "USD"); + } +} From 70a78a9b41f6609f3bb784da6b58cf25e5187526 Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 15 Feb 2025 11:40:26 +0200 Subject: [PATCH 67/89] Fixes --- my-no-sql-tests/src/tests_from_real_life.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/my-no-sql-tests/src/tests_from_real_life.rs b/my-no-sql-tests/src/tests_from_real_life.rs index 1019529..1a1788d 100644 --- a/my-no-sql-tests/src/tests_from_real_life.rs +++ b/my-no-sql-tests/src/tests_from_real_life.rs @@ -12,7 +12,9 @@ pub struct PayoutWithdrawalSettingsMyNoSqlEntity { #[cfg(test)] mod tests { - use my_no_sql_sdk::abstractions::MyNoSqlEntitySerializer; + use my_no_sql_sdk::{ + abstractions::MyNoSqlEntitySerializer, core::rust_extensions::date_time::DateTimeStruct, + }; use super::PayoutWithdrawalSettingsMyNoSqlEntity; @@ -25,5 +27,18 @@ mod tests { assert_eq!(entity.value, 15000.0); assert_eq!(entity.currency, "USD"); + assert_eq!(entity.partition_key, "bank-transfer"); + assert_eq!(entity.row_key, "max"); + + let dt_struct: DateTimeStruct = entity.time_stamp.to_date_time().into(); + + assert_eq!(dt_struct.year, 2024); + assert_eq!(dt_struct.month, 11); + assert_eq!(dt_struct.day, 29); + + assert_eq!(dt_struct.time.hour, 14); + assert_eq!(dt_struct.time.min, 59); + assert_eq!(dt_struct.time.sec, 15); + assert_eq!(dt_struct.time.micros, 614500); } } From e2c3c5f0d8117201618d0ff49b0ff3a5660328f1 Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 8 Mar 2025 11:05:54 +0100 Subject: [PATCH 68/89] Added server-core --- Cargo.toml | 1 + my-no-sql-core/Cargo.toml | 3 +- my-no-sql-core/src/lib.rs | 3 + my-no-sql-server-core/Cargo.toml | 17 ++++ my-no-sql-server-core/src/db_instance.rs | 50 ++++++++++ .../src/db_snapshots/db_partition_snapshot.rs | 41 ++++++++ .../db_rows_by_partitions_snapshot.rs | 61 ++++++++++++ .../src/db_snapshots/db_rows_snapshot.rs | 43 ++++++++ .../src/db_snapshots/db_table_snapshot.rs | 47 +++++++++ my-no-sql-server-core/src/db_snapshots/mod.rs | 10 ++ my-no-sql-server-core/src/db_table_wrapper.rs | 98 +++++++++++++++++++ my-no-sql-server-core/src/lib.rs | 7 ++ 12 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 my-no-sql-server-core/Cargo.toml create mode 100644 my-no-sql-server-core/src/db_instance.rs create mode 100644 my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs create mode 100644 my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs create mode 100644 my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs create mode 100644 my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs create mode 100644 my-no-sql-server-core/src/db_snapshots/mod.rs create mode 100644 my-no-sql-server-core/src/db_table_wrapper.rs create mode 100644 my-no-sql-server-core/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 2d8ed6d..aae083b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,5 @@ members = [ "my-no-sql-sdk", "my-no-sql-core", "my-no-sql-tests", + "my-no-sql-server-core", ] diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index 4e9e9be..4ad2ac3 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [features] default = [] -master-node = [] +master-node = ["my-no-sql-server-core"] debug_db_row = [] @@ -13,6 +13,7 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } +my-no-sql-server-core = { path = "../my-no-sql-server-core", optional = true } my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } diff --git a/my-no-sql-core/src/lib.rs b/my-no-sql-core/src/lib.rs index c5aa256..dbf7db9 100644 --- a/my-no-sql-core/src/lib.rs +++ b/my-no-sql-core/src/lib.rs @@ -7,3 +7,6 @@ pub use expiration_index::*; pub mod entity_serializer; pub extern crate my_json; pub extern crate rust_extensions; + +#[cfg(feature = "master-node")] +pub extern crate my_no_sql_server_core as server; diff --git a/my-no-sql-server-core/Cargo.toml b/my-no-sql-server-core/Cargo.toml new file mode 100644 index 0000000..f6cd628 --- /dev/null +++ b/my-no-sql-server-core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "my-no-sql-server-core" +version = "0.4.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = [] +master-node = ["my-no-sql-sdk/master-node"] + +[dependencies] +rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ + "with-tokio", + "base64", +] } +tokio = { version = "*", features = ["full"] } +my-no-sql-sdk = { tag = "0.4.1", git = "https://github.com/MyJetTools/my-no-sql-sdk.git" } diff --git a/my-no-sql-server-core/src/db_instance.rs b/my-no-sql-server-core/src/db_instance.rs new file mode 100644 index 0000000..c839443 --- /dev/null +++ b/my-no-sql-server-core/src/db_instance.rs @@ -0,0 +1,50 @@ +use my_no_sql_sdk::core::db::DbTableName; +use tokio::sync::RwLock; + +use std::{collections::HashMap, sync::Arc}; + +use super::DbTableWrapper; + +pub struct DbInstance { + pub tables: RwLock>>, +} + +impl DbInstance { + pub fn new() -> DbInstance { + DbInstance { + tables: RwLock::new(HashMap::new()), + } + } + + pub async fn get_table_names(&self) -> Vec { + let read_access = self.tables.read().await; + + return read_access + .values() + .into_iter() + .map(|table| table.name.clone()) + .collect(); + } + + pub async fn get_tables(&self) -> Vec> { + let read_access = self.tables.read().await; + + return read_access + .values() + .into_iter() + .map(|table| table.clone()) + .collect(); + } + + pub async fn get_table(&self, table_name: &str) -> Option> { + let read_access = self.tables.read().await; + + let result = read_access.get(table_name)?; + return Some(result.clone()); + } + + pub async fn delete_table(&self, table_name: &str) -> Option> { + let mut write_access = self.tables.write().await; + write_access.remove(table_name) + } +} diff --git a/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs new file mode 100644 index 0000000..7342380 --- /dev/null +++ b/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs @@ -0,0 +1,41 @@ +use my_no_sql_sdk::core::db::{DbPartition, PartitionKey}; + +use super::DbRowsSnapshot; + +pub struct DbPartitionSnapshot { + #[cfg(feature = "master-node")] + pub last_read_moment: rust_extensions::date_time::DateTimeAsMicroseconds, + #[cfg(feature = "master-node")] + pub last_write_moment: rust_extensions::date_time::DateTimeAsMicroseconds, + pub partition_key: PartitionKey, + pub db_rows_snapshot: DbRowsSnapshot, +} + +#[cfg(feature = "master-node")] +impl DbPartitionSnapshot { + pub fn has_to_persist( + &self, + written_in_blob: rust_extensions::date_time::DateTimeAsMicroseconds, + ) -> bool { + written_in_blob.unix_microseconds < self.last_write_moment.unix_microseconds + } +} + +impl Into for &DbPartition { + fn into(self) -> DbRowsSnapshot { + DbRowsSnapshot::new_from_snapshot(self.rows.get_all().map(|itm| itm.clone()).collect()) + } +} + +impl Into for &DbPartition { + fn into(self) -> DbPartitionSnapshot { + DbPartitionSnapshot { + #[cfg(feature = "master-node")] + last_read_moment: self.last_read_moment.as_date_time(), + #[cfg(feature = "master-node")] + last_write_moment: self.last_write_moment, + partition_key: self.partition_key.clone(), + db_rows_snapshot: self.into(), + } + } +} diff --git a/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs new file mode 100644 index 0000000..9909df3 --- /dev/null +++ b/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs @@ -0,0 +1,61 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::{DbRow, PartitionKey, PartitionKeyParameter}; +use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; + +pub struct DbRowsByPartitionsSnapshot { + pub partitions: Vec<(PartitionKey, Vec>)>, +} + +impl DbRowsByPartitionsSnapshot { + pub fn new() -> Self { + Self { + partitions: Vec::new(), + } + } + + pub fn has_elements(&self) -> bool { + self.partitions.len() > 0 + } + + fn get_or_create_partition( + &mut self, + partition_key: impl PartitionKeyParameter, + ) -> &mut Vec> { + let index = self + .partitions + .binary_search_by(|itm| itm.0.as_str().cmp(partition_key.as_str())); + + match index { + Ok(index) => self.partitions.get_mut(index).unwrap().1.as_mut(), + Err(index) => { + self.partitions + .insert(index, (partition_key.to_partition_key(), Vec::new())); + self.partitions.get_mut(index).unwrap().1.as_mut() + } + } + } + + pub fn add_row(&mut self, partition_key: impl PartitionKeyParameter, db_row: Arc) { + self.get_or_create_partition(partition_key).push(db_row); + } + + pub fn add_rows( + &mut self, + partition_key: impl PartitionKeyParameter, + db_rows: Vec>, + ) { + self.get_or_create_partition(partition_key).extend(db_rows); + } + + pub fn as_json_array(&self) -> JsonArrayWriter { + let mut json_array_writer = JsonArrayWriter::new(); + for (_, snapshot) in self.partitions.iter() { + for db_row in snapshot { + json_array_writer.write(db_row.as_ref()); + } + } + + json_array_writer + } +} diff --git a/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs new file mode 100644 index 0000000..a7c9e86 --- /dev/null +++ b/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs @@ -0,0 +1,43 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::DbRow; +use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; + +pub struct DbRowsSnapshot { + pub db_rows: Vec>, +} + +impl DbRowsSnapshot { + pub fn new() -> Self { + Self { + db_rows: Vec::new(), + } + } + + pub fn new_from_snapshot(db_rows: Vec>) -> Self { + Self { db_rows } + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + db_rows: Vec::with_capacity(capacity), + } + } + + pub fn push(&mut self, db_row: Arc) { + self.db_rows.push(db_row); + } + + pub fn len(&self) -> usize { + self.db_rows.len() + } + + pub fn as_json_array(&self) -> JsonArrayWriter { + let mut json_array_writer = JsonArrayWriter::new(); + for db_row in &self.db_rows { + json_array_writer.write(db_row.as_ref()); + } + + json_array_writer + } +} diff --git a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs new file mode 100644 index 0000000..bcc0acd --- /dev/null +++ b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs @@ -0,0 +1,47 @@ +use my_no_sql_sdk::core::db::DbTable; +use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; +#[cfg(feature = "master-node")] +use rust_extensions::date_time::DateTimeAsMicroseconds; + +use super::DbPartitionSnapshot; + +pub struct DbTableSnapshot { + #[cfg(feature = "master-node")] + pub attr: my_no_sql_sdk::core::db::DbTableAttributes, + #[cfg(feature = "master-node")] + pub last_write_moment: DateTimeAsMicroseconds, + pub by_partition: Vec, +} + +impl DbTableSnapshot { + pub fn new( + #[cfg(feature = "master-node")] last_write_moment: DateTimeAsMicroseconds, + db_table: &DbTable, + ) -> Self { + let mut by_partition = Vec::new(); + + for db_partition in db_table.partitions.get_partitions() { + by_partition.push(db_partition.into()); + } + + Self { + #[cfg(feature = "master-node")] + attr: db_table.attributes.clone(), + #[cfg(feature = "master-node")] + last_write_moment, + by_partition, + } + } + + pub fn as_json_array(&self) -> JsonArrayWriter { + let mut json_array_writer = JsonArrayWriter::new(); + + for db_partition_snapshot in self.by_partition.iter() { + for db_row in &db_partition_snapshot.db_rows_snapshot.db_rows { + json_array_writer.write(db_row.as_ref()); + } + } + + json_array_writer + } +} diff --git a/my-no-sql-server-core/src/db_snapshots/mod.rs b/my-no-sql-server-core/src/db_snapshots/mod.rs new file mode 100644 index 0000000..fdcc05a --- /dev/null +++ b/my-no-sql-server-core/src/db_snapshots/mod.rs @@ -0,0 +1,10 @@ +mod db_partition_snapshot; +mod db_rows_by_partitions_snapshot; +mod db_rows_snapshot; + +mod db_table_snapshot; +pub use db_partition_snapshot::DbPartitionSnapshot; +pub use db_rows_by_partitions_snapshot::DbRowsByPartitionsSnapshot; +pub use db_rows_snapshot::DbRowsSnapshot; + +pub use db_table_snapshot::DbTableSnapshot; diff --git a/my-no-sql-server-core/src/db_table_wrapper.rs b/my-no-sql-server-core/src/db_table_wrapper.rs new file mode 100644 index 0000000..2f21ca0 --- /dev/null +++ b/my-no-sql-server-core/src/db_table_wrapper.rs @@ -0,0 +1,98 @@ +use std::{collections::VecDeque, sync::Arc}; + +use crate::db_snapshots::{DbPartitionSnapshot, DbTableSnapshot}; +use my_no_sql_sdk::core::db::{DbRow, DbTable, DbTableName}; +use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; +use tokio::sync::RwLock; + +#[cfg(feature = "master-node")] +use my_no_sql_sdk::core::db::DbTableAttributes; + +pub struct DbTableWrapper { + pub name: DbTableName, + pub data: RwLock, +} + +impl DbTableWrapper { + pub fn new(db_table: DbTable) -> Arc { + let result = Self { + name: db_table.name.clone(), + data: RwLock::new(db_table), + }; + + Arc::new(result) + } + + pub async fn get_table_as_json_array(&self) -> JsonArrayWriter { + let read_access = self.data.read().await; + read_access.get_table_as_json_array() + } + + pub async fn get_all_as_vec_dequeue(&self) -> VecDeque> { + let read_access = self.data.read().await; + + let mut result = VecDeque::new(); + + for (_, db_row) in read_access.get_all_rows(None, None) { + result.push_back(db_row.clone()); + } + + result + } + + pub async fn get_table_snapshot(&self) -> DbTableSnapshot { + let read_access = self.data.read().await; + + DbTableSnapshot { + #[cfg(feature = "master-node")] + last_write_moment: read_access.get_last_write_moment(), + by_partition: get_partitions_snapshot(&read_access), + #[cfg(feature = "master-node")] + attr: read_access.attributes.clone(), + } + } + + pub async fn get_partitions_amount(&self) -> usize { + let read_access = self.data.read().await; + read_access.partitions.len() + } + #[cfg(feature = "master-node")] + pub async fn get_persist_table(&self) -> bool { + let read_access = self.data.read().await; + read_access.attributes.persist + } + + pub async fn get_table_size(&self) -> usize { + let read_access = self.data.read().await; + read_access.get_table_size() + } + + #[cfg(feature = "master-node")] + pub async fn get_max_partitions_amount(&self) -> Option { + let read_access = self.data.read().await; + read_access.attributes.max_partitions_amount + } + + #[cfg(feature = "master-node")] + pub async fn get_attributes(&self) -> DbTableAttributes { + let read_access = self.data.read().await; + read_access.attributes.clone() + } + + #[cfg(feature = "master-node")] + pub async fn get_partition_snapshot(&self, partition_key: &str) -> Option { + let read_access = self.data.read().await; + let db_partition = read_access.get_partition(partition_key)?; + Some(db_partition.into()) + } +} + +fn get_partitions_snapshot(db_table: &DbTable) -> Vec { + let mut result = Vec::with_capacity(db_table.partitions.len()); + + for db_partition in db_table.partitions.get_partitions() { + result.push(db_partition.into()); + } + + result +} diff --git a/my-no-sql-server-core/src/lib.rs b/my-no-sql-server-core/src/lib.rs new file mode 100644 index 0000000..486b092 --- /dev/null +++ b/my-no-sql-server-core/src/lib.rs @@ -0,0 +1,7 @@ +mod db_instance; +mod db_table_wrapper; +pub use db_instance::*; +pub use db_table_wrapper::*; + +pub mod db_snapshots; +pub extern crate rust_extensions; From 734e33794daec2ab2fbf3eb7a6d07eeda51649a5 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 11 Mar 2025 19:23:21 +0100 Subject: [PATCH 69/89] Now No Panic on Somehow we did not find the index --- my-no-sql-core/src/expiration_index.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/my-no-sql-core/src/expiration_index.rs b/my-no-sql-core/src/expiration_index.rs index 748e0d4..8ac39c1 100644 --- a/my-no-sql-core/src/expiration_index.rs +++ b/my-no-sql-core/src/expiration_index.rs @@ -107,16 +107,16 @@ impl> ExpirationIndexContainer { - panic!( + println!( "Somehow we did not find the index for expiration moment {} of '{}'. Expiration moment as rfc3339 is {}", expiration_moment.unix_microseconds, key_as_str, expiration_moment.to_rfc3339() ); } } - - self.amount -= 1; } pub fn get_items_to_expire( From 2b2f50145cb4b5b941ecfd3e29a20f4cf3618485 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 11 Mar 2025 19:28:57 +0100 Subject: [PATCH 70/89] Removed server-core for a while --- Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aae083b..6d86dc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,5 +8,5 @@ members = [ "my-no-sql-sdk", "my-no-sql-core", "my-no-sql-tests", - "my-no-sql-server-core", + # "my-no-sql-server-core", ] diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index 4ad2ac3..1048959 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [features] default = [] -master-node = ["my-no-sql-server-core"] +master-node = [] debug_db_row = [] @@ -13,7 +13,7 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -my-no-sql-server-core = { path = "../my-no-sql-server-core", optional = true } +#my-no-sql-server-core = { path = "../my-no-sql-server-core", optional = true } my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } From 44362701ac292b41b5be0b7d86d0acb0dc2197a7 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 11 Mar 2025 19:39:19 +0100 Subject: [PATCH 71/89] Now SDK has server-core as well --- Cargo.toml | 2 +- my-no-sql-core/Cargo.toml | 1 - my-no-sql-core/src/expiration_index.rs | 6 +++--- my-no-sql-core/src/lib.rs | 3 --- my-no-sql-sdk/Cargo.toml | 4 +++- my-no-sql-sdk/src/lib.rs | 3 +++ my-no-sql-server-core/Cargo.toml | 4 ++-- my-no-sql-server-core/src/db_instance.rs | 2 +- .../src/db_snapshots/db_partition_snapshot.rs | 2 +- .../src/db_snapshots/db_rows_by_partitions_snapshot.rs | 4 ++-- my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs | 4 ++-- my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs | 4 ++-- my-no-sql-server-core/src/db_table_wrapper.rs | 4 ++-- 13 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6d86dc6..aae083b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,5 +8,5 @@ members = [ "my-no-sql-sdk", "my-no-sql-core", "my-no-sql-tests", - # "my-no-sql-server-core", + "my-no-sql-server-core", ] diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index 1048959..4e9e9be 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -13,7 +13,6 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -#my-no-sql-server-core = { path = "../my-no-sql-server-core", optional = true } my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } diff --git a/my-no-sql-core/src/expiration_index.rs b/my-no-sql-core/src/expiration_index.rs index 8ac39c1..748e0d4 100644 --- a/my-no-sql-core/src/expiration_index.rs +++ b/my-no-sql-core/src/expiration_index.rs @@ -107,16 +107,16 @@ impl> ExpirationIndexContainer { - println!( + panic!( "Somehow we did not find the index for expiration moment {} of '{}'. Expiration moment as rfc3339 is {}", expiration_moment.unix_microseconds, key_as_str, expiration_moment.to_rfc3339() ); } } + + self.amount -= 1; } pub fn get_items_to_expire( diff --git a/my-no-sql-core/src/lib.rs b/my-no-sql-core/src/lib.rs index dbf7db9..c5aa256 100644 --- a/my-no-sql-core/src/lib.rs +++ b/my-no-sql-core/src/lib.rs @@ -7,6 +7,3 @@ pub use expiration_index::*; pub mod entity_serializer; pub extern crate my_json; pub extern crate rust_extensions; - -#[cfg(feature = "master-node")] -pub extern crate my_no_sql_server_core as server; diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index 54f2aba..423b7ab 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -11,14 +11,16 @@ data-writer = ["dep:my-no-sql-data-writer"] macros = ["dep:my-no-sql-macros", "dep:rust-extensions"] data-reader = ["dep:my-no-sql-tcp-reader"] tcp-contracts = ["dep:my-no-sql-tcp-shared"] -master-node = ["my-no-sql-core/master-node"] +master-node = ["my-no-sql-core/master-node", "my-no-sql-server-core"] debug_db_row = ["my-no-sql-core/debug_db_row"] with-ssh = ["my-no-sql-data-writer?/with-ssh"] + [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } my-no-sql-core = { path = "../my-no-sql-core" } +my-no-sql-server-core = { path = "../my-no-sql-server-core", optional = true } my-no-sql-data-writer = { optional = true, path = "../my-no-sql-data-writer" } my-no-sql-macros = { optional = true, path = "../my-no-sql-macros" } diff --git a/my-no-sql-sdk/src/lib.rs b/my-no-sql-sdk/src/lib.rs index e77b4a7..92afbb3 100644 --- a/my-no-sql-sdk/src/lib.rs +++ b/my-no-sql-sdk/src/lib.rs @@ -15,3 +15,6 @@ pub extern crate my_no_sql_tcp_reader as reader; #[cfg(feature = "tcp-contracts")] pub extern crate my_no_sql_tcp_shared as tcp_contracts; + +#[cfg(feature = "master-node")] +pub extern crate my_no_sql_server_core as server; diff --git a/my-no-sql-server-core/Cargo.toml b/my-no-sql-server-core/Cargo.toml index f6cd628..462dfeb 100644 --- a/my-no-sql-server-core/Cargo.toml +++ b/my-no-sql-server-core/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] default = [] -master-node = ["my-no-sql-sdk/master-node"] +master-node = ["my-no-sql-core/master-node"] [dependencies] rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ @@ -14,4 +14,4 @@ rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-ext "base64", ] } tokio = { version = "*", features = ["full"] } -my-no-sql-sdk = { tag = "0.4.1", git = "https://github.com/MyJetTools/my-no-sql-sdk.git" } +my-no-sql-core = { path = "../my-no-sql-core" } diff --git a/my-no-sql-server-core/src/db_instance.rs b/my-no-sql-server-core/src/db_instance.rs index c839443..43ea64f 100644 --- a/my-no-sql-server-core/src/db_instance.rs +++ b/my-no-sql-server-core/src/db_instance.rs @@ -1,4 +1,4 @@ -use my_no_sql_sdk::core::db::DbTableName; +use my_no_sql_core::db::DbTableName; use tokio::sync::RwLock; use std::{collections::HashMap, sync::Arc}; diff --git a/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs index 7342380..d2a6e88 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_partition_snapshot.rs @@ -1,4 +1,4 @@ -use my_no_sql_sdk::core::db::{DbPartition, PartitionKey}; +use my_no_sql_core::db::{DbPartition, PartitionKey}; use super::DbRowsSnapshot; diff --git a/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs index 9909df3..dec7bb8 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use my_no_sql_sdk::core::db::{DbRow, PartitionKey, PartitionKeyParameter}; -use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; +use my_no_sql_core::db::{DbRow, PartitionKey, PartitionKeyParameter}; +use my_no_sql_core::my_json::json_writer::JsonArrayWriter; pub struct DbRowsByPartitionsSnapshot { pub partitions: Vec<(PartitionKey, Vec>)>, diff --git a/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs index a7c9e86..5f22325 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use my_no_sql_sdk::core::db::DbRow; -use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; +use my_no_sql_core::db::DbRow; +use my_no_sql_core::my_json::json_writer::JsonArrayWriter; pub struct DbRowsSnapshot { pub db_rows: Vec>, diff --git a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs index bcc0acd..4920c71 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs @@ -1,5 +1,5 @@ -use my_no_sql_sdk::core::db::DbTable; -use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; +use my_no_sql_core::db::DbTable; +use my_no_sql_core::my_json::json_writer::JsonArrayWriter; #[cfg(feature = "master-node")] use rust_extensions::date_time::DateTimeAsMicroseconds; diff --git a/my-no-sql-server-core/src/db_table_wrapper.rs b/my-no-sql-server-core/src/db_table_wrapper.rs index 2f21ca0..7753c8a 100644 --- a/my-no-sql-server-core/src/db_table_wrapper.rs +++ b/my-no-sql-server-core/src/db_table_wrapper.rs @@ -1,8 +1,8 @@ use std::{collections::VecDeque, sync::Arc}; use crate::db_snapshots::{DbPartitionSnapshot, DbTableSnapshot}; -use my_no_sql_sdk::core::db::{DbRow, DbTable, DbTableName}; -use my_no_sql_sdk::core::my_json::json_writer::JsonArrayWriter; +use my_no_sql_core::db::*; +use my_no_sql_core::my_json::json_writer::JsonArrayWriter; use tokio::sync::RwLock; #[cfg(feature = "master-node")] From c16637a15a7e53aac33034946ac57abed2288a65 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 11 Mar 2025 19:46:06 +0100 Subject: [PATCH 72/89] Fixes --- my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs | 2 +- my-no-sql-server-core/src/db_table_wrapper.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs index 4920c71..4fa6ba4 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs @@ -7,7 +7,7 @@ use super::DbPartitionSnapshot; pub struct DbTableSnapshot { #[cfg(feature = "master-node")] - pub attr: my_no_sql_sdk::core::db::DbTableAttributes, + pub attr: my_no_sql_core::db::DbTableAttributes, #[cfg(feature = "master-node")] pub last_write_moment: DateTimeAsMicroseconds, pub by_partition: Vec, diff --git a/my-no-sql-server-core/src/db_table_wrapper.rs b/my-no-sql-server-core/src/db_table_wrapper.rs index 7753c8a..b03f6bd 100644 --- a/my-no-sql-server-core/src/db_table_wrapper.rs +++ b/my-no-sql-server-core/src/db_table_wrapper.rs @@ -6,7 +6,7 @@ use my_no_sql_core::my_json::json_writer::JsonArrayWriter; use tokio::sync::RwLock; #[cfg(feature = "master-node")] -use my_no_sql_sdk::core::db::DbTableAttributes; +use my_no_sql_core::db::DbTableAttributes; pub struct DbTableWrapper { pub name: DbTableName, From df664f108dc3f9349f0e7f4ea0079c47c8a635e1 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 11 Mar 2025 20:05:28 +0100 Subject: [PATCH 73/89] Fixes --- my-no-sql-sdk/Cargo.toml | 5 ++++- my-no-sql-server-core/Cargo.toml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index 423b7ab..dbb4b17 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -11,7 +11,10 @@ data-writer = ["dep:my-no-sql-data-writer"] macros = ["dep:my-no-sql-macros", "dep:rust-extensions"] data-reader = ["dep:my-no-sql-tcp-reader"] tcp-contracts = ["dep:my-no-sql-tcp-shared"] -master-node = ["my-no-sql-core/master-node", "my-no-sql-server-core"] +master-node = [ + "my-no-sql-core/master-node", + "my-no-sql-server-core/master-node", +] debug_db_row = ["my-no-sql-core/debug_db_row"] with-ssh = ["my-no-sql-data-writer?/with-ssh"] diff --git a/my-no-sql-server-core/Cargo.toml b/my-no-sql-server-core/Cargo.toml index 462dfeb..18df88f 100644 --- a/my-no-sql-server-core/Cargo.toml +++ b/my-no-sql-server-core/Cargo.toml @@ -12,6 +12,7 @@ master-node = ["my-no-sql-core/master-node"] rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git", features = [ "with-tokio", "base64", + ] } tokio = { version = "*", features = ["full"] } my-no-sql-core = { path = "../my-no-sql-core" } From 10870552faa0d9c4dbe87574d744535225fe4c49 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 11 Mar 2025 20:27:30 +0100 Subject: [PATCH 74/89] Fixes --- my-no-sql-core/src/db/db_table/db_table.rs | 8 ++++---- my-no-sql-core/src/db/db_table/db_table_master_node.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/my-no-sql-core/src/db/db_table/db_table.rs b/my-no-sql-core/src/db/db_table/db_table.rs index 35203ae..b720d37 100644 --- a/my-no-sql-core/src/db/db_table/db_table.rs +++ b/my-no-sql-core/src/db/db_table/db_table.rs @@ -128,14 +128,14 @@ impl DbTable { #[inline] pub fn insert_or_replace_row( &mut self, - db_row: &Arc, + db_row: Arc, #[cfg(feature = "master-node")] set_last_write_moment: Option, ) -> (PartitionKey, Option>) { - self.avg_size.add(db_row); + self.avg_size.add(&db_row); - let db_partition = self.partitions.add_partition_if_not_exists(db_row); + let db_partition = self.partitions.add_partition_if_not_exists(&db_row); - let removed_db_row = db_partition.insert_or_replace_row(db_row.clone()); + let removed_db_row = db_partition.insert_or_replace_row(db_row); #[cfg(feature = "master-node")] if let Some(set_last_write_moment) = set_last_write_moment { diff --git a/my-no-sql-core/src/db/db_table/db_table_master_node.rs b/my-no-sql-core/src/db/db_table/db_table_master_node.rs index 87f0b88..c3a2199 100644 --- a/my-no-sql-core/src/db/db_table/db_table_master_node.rs +++ b/my-no-sql-core/src/db/db_table/db_table_master_node.rs @@ -159,7 +159,7 @@ mod tests { let db_row2 = Arc::new(db_row2); - db_table.insert_or_replace_row(&db_row2, None); + db_table.insert_or_replace_row(db_row2.clone(), None); assert_eq!(db_table.get_table_size(), db_row2.get_src_as_slice().len()); assert_eq!(db_table.get_partitions_amount(), 1); From 122c8ec22f0b8c74915bec993efa343b37a516dd Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 12 Mar 2025 09:28:07 +0100 Subject: [PATCH 75/89] Removed --- my-no-sql-core/src/expiration_index.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/my-no-sql-core/src/expiration_index.rs b/my-no-sql-core/src/expiration_index.rs index 748e0d4..0a71c6e 100644 --- a/my-no-sql-core/src/expiration_index.rs +++ b/my-no-sql-core/src/expiration_index.rs @@ -107,16 +107,17 @@ impl> ExpirationIndexContainer { - panic!( + //todo!("Removed - but I have to return it"); + println!( "Somehow we did not find the index for expiration moment {} of '{}'. Expiration moment as rfc3339 is {}", expiration_moment.unix_microseconds, key_as_str, expiration_moment.to_rfc3339() ); } } - - self.amount -= 1; } pub fn get_items_to_expire( From f6742060bac4ee735bbf168d273ed5e7f8ce3578 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 12 Mar 2025 09:28:18 +0100 Subject: [PATCH 76/89] Fixes --- my-no-sql-core/src/expiration_index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my-no-sql-core/src/expiration_index.rs b/my-no-sql-core/src/expiration_index.rs index 0a71c6e..250ddb9 100644 --- a/my-no-sql-core/src/expiration_index.rs +++ b/my-no-sql-core/src/expiration_index.rs @@ -111,7 +111,7 @@ impl> ExpirationIndexContainer { - //todo!("Removed - but I have to return it"); + //todo!("Removed - but I have to return it. At least unit test it"); println!( "Somehow we did not find the index for expiration moment {} of '{}'. Expiration moment as rfc3339 is {}", expiration_moment.unix_microseconds, key_as_str, expiration_moment.to_rfc3339() From c75e2bc7aac2e991f50b03fb4e6085339be7c9dd Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 12 Mar 2025 21:41:23 +0100 Subject: [PATCH 77/89] Renaming DbTableInner --- my-no-sql-core/src/db/db_table/db_table.rs | 10 +++++----- .../src/db/db_table/db_table_master_node.rs | 14 +++++++++----- .../src/db_snapshots/db_table_snapshot.rs | 4 ++-- my-no-sql-server-core/src/db_table_wrapper.rs | 6 +++--- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/my-no-sql-core/src/db/db_table/db_table.rs b/my-no-sql-core/src/db/db_table/db_table.rs index b720d37..52459ef 100644 --- a/my-no-sql-core/src/db/db_table/db_table.rs +++ b/my-no-sql-core/src/db/db_table/db_table.rs @@ -10,7 +10,7 @@ use crate::db::{DbPartition, DbRow, PartitionKey, PartitionKeyParameter, RowKeyP use super::DbTableAttributes; use super::{AllDbRowsIterator, AvgSize, ByRowKeyIterator, DbPartitionsContainer, DbTableName}; -pub struct DbTable { +pub struct DbTableInner { pub name: DbTableName, pub partitions: DbPartitionsContainer, pub avg_size: AvgSize, @@ -20,13 +20,13 @@ pub struct DbTable { pub attributes: DbTableAttributes, } -impl EntityWithStrKey for DbTable { +impl EntityWithStrKey for DbTableInner { fn get_key(&self) -> &str { self.name.as_str() } } -impl DbTable { +impl DbTableInner { #[cfg(not(feature = "master-node"))] pub fn new(name: DbTableName) -> Self { Self { @@ -124,7 +124,7 @@ impl DbTable { /// Insert Operations -impl DbTable { +impl DbTableInner { #[inline] pub fn insert_or_replace_row( &mut self, @@ -204,7 +204,7 @@ impl DbTable { /// /// -impl DbTable { +impl DbTableInner { pub fn remove_row( &mut self, partition_key: &impl PartitionKeyParameter, diff --git a/my-no-sql-core/src/db/db_table/db_table_master_node.rs b/my-no-sql-core/src/db/db_table/db_table_master_node.rs index c3a2199..eefd5af 100644 --- a/my-no-sql-core/src/db/db_table/db_table_master_node.rs +++ b/my-no-sql-core/src/db/db_table/db_table_master_node.rs @@ -5,7 +5,9 @@ use rust_extensions::{ use crate::db::PartitionKey; -use super::{AvgSize, DataToGc, DbPartitionsContainer, DbTable, DbTableAttributes, DbTableName}; +use super::{ + AvgSize, DataToGc, DbPartitionsContainer, DbTableAttributes, DbTableInner, DbTableName, +}; pub struct PartitionLastWriteMoment { pub partition_key: PartitionKey, @@ -18,7 +20,7 @@ impl EntityWithStrKey for PartitionLastWriteMoment { } } -impl DbTable { +impl DbTableInner { pub fn new(name: DbTableName, attributes: DbTableAttributes) -> Self { Self { name, @@ -105,7 +107,7 @@ mod tests { use std::sync::Arc; use crate::{ - db::DbTable, + db::DbTableInner, db_json_entity::{DbJsonEntity, JsonTimeStamp}, }; @@ -113,7 +115,8 @@ mod tests { #[test] fn test_insert_record() { - let mut db_table = DbTable::new("test-table".into(), DbTableAttributes::create_default()); + let mut db_table = + DbTableInner::new("test-table".into(), DbTableAttributes::create_default()); let now = JsonTimeStamp::now(); @@ -134,7 +137,8 @@ mod tests { #[test] fn test_insert_and_insert_or_replace() { - let mut db_table = DbTable::new("test-table".into(), DbTableAttributes::create_default()); + let mut db_table = + DbTableInner::new("test-table".into(), DbTableAttributes::create_default()); let now = JsonTimeStamp::now(); diff --git a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs index 4fa6ba4..b245771 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs @@ -1,4 +1,4 @@ -use my_no_sql_core::db::DbTable; +use my_no_sql_core::db::DbTableInner; use my_no_sql_core::my_json::json_writer::JsonArrayWriter; #[cfg(feature = "master-node")] use rust_extensions::date_time::DateTimeAsMicroseconds; @@ -16,7 +16,7 @@ pub struct DbTableSnapshot { impl DbTableSnapshot { pub fn new( #[cfg(feature = "master-node")] last_write_moment: DateTimeAsMicroseconds, - db_table: &DbTable, + db_table: &DbTableInner, ) -> Self { let mut by_partition = Vec::new(); diff --git a/my-no-sql-server-core/src/db_table_wrapper.rs b/my-no-sql-server-core/src/db_table_wrapper.rs index b03f6bd..5584469 100644 --- a/my-no-sql-server-core/src/db_table_wrapper.rs +++ b/my-no-sql-server-core/src/db_table_wrapper.rs @@ -10,11 +10,11 @@ use my_no_sql_core::db::DbTableAttributes; pub struct DbTableWrapper { pub name: DbTableName, - pub data: RwLock, + pub data: RwLock, } impl DbTableWrapper { - pub fn new(db_table: DbTable) -> Arc { + pub fn new(db_table: DbTableInner) -> Arc { let result = Self { name: db_table.name.clone(), data: RwLock::new(db_table), @@ -87,7 +87,7 @@ impl DbTableWrapper { } } -fn get_partitions_snapshot(db_table: &DbTable) -> Vec { +fn get_partitions_snapshot(db_table: &DbTableInner) -> Vec { let mut result = Vec::with_capacity(db_table.partitions.len()); for db_partition in db_table.partitions.get_partitions() { From 80097dddce5bd1439fc63e0cc8c870c72bd5526b Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 12 Mar 2025 21:51:09 +0100 Subject: [PATCH 78/89] Added read-node --- my-no-sql-sdk/Cargo.toml | 3 +++ my-no-sql-sdk/src/lib.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/my-no-sql-sdk/Cargo.toml b/my-no-sql-sdk/Cargo.toml index dbb4b17..dc1b8f0 100644 --- a/my-no-sql-sdk/Cargo.toml +++ b/my-no-sql-sdk/Cargo.toml @@ -15,6 +15,9 @@ master-node = [ "my-no-sql-core/master-node", "my-no-sql-server-core/master-node", ] + +read-node = ["my-no-sql-server-core"] + debug_db_row = ["my-no-sql-core/debug_db_row"] with-ssh = ["my-no-sql-data-writer?/with-ssh"] diff --git a/my-no-sql-sdk/src/lib.rs b/my-no-sql-sdk/src/lib.rs index 92afbb3..1e9206d 100644 --- a/my-no-sql-sdk/src/lib.rs +++ b/my-no-sql-sdk/src/lib.rs @@ -18,3 +18,6 @@ pub extern crate my_no_sql_tcp_shared as tcp_contracts; #[cfg(feature = "master-node")] pub extern crate my_no_sql_server_core as server; + +#[cfg(feature = "read-node")] +pub extern crate my_no_sql_server_core as server; From eab7d52d8bbfa1adc3aca3aa59d8414213c02a85 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 12 Mar 2025 22:29:02 +0100 Subject: [PATCH 79/89] Now DbTableWrapper is DbTable --- .../src/db/db_partition/db_rows_container.rs | 2 +- .../src/db/db_table/{db_table.rs => db_table_inner.rs} | 0 my-no-sql-core/src/db/db_table/mod.rs | 4 ++-- my-no-sql-sdk/src/lib.rs | 5 +---- my-no-sql-server-core/src/db_instance.rs | 10 +++++----- .../src/{db_table_wrapper.rs => db_table.rs} | 4 ++-- my-no-sql-server-core/src/lib.rs | 4 ++-- 7 files changed, 13 insertions(+), 16 deletions(-) rename my-no-sql-core/src/db/db_table/{db_table.rs => db_table_inner.rs} (100%) rename my-no-sql-server-core/src/{db_table_wrapper.rs => db_table.rs} (98%) diff --git a/my-no-sql-core/src/db/db_partition/db_rows_container.rs b/my-no-sql-core/src/db/db_partition/db_rows_container.rs index 810e2fd..bc7a5c6 100644 --- a/my-no-sql-core/src/db/db_partition/db_rows_container.rs +++ b/my-no-sql-core/src/db/db_partition/db_rows_container.rs @@ -152,7 +152,7 @@ fn are_expires_the_same( #[cfg(feature = "master-node")] #[cfg(test)] -mod tests { +mod expiration_tests { use rust_extensions::date_time::DateTimeAsMicroseconds; diff --git a/my-no-sql-core/src/db/db_table/db_table.rs b/my-no-sql-core/src/db/db_table/db_table_inner.rs similarity index 100% rename from my-no-sql-core/src/db/db_table/db_table.rs rename to my-no-sql-core/src/db/db_table/db_table_inner.rs diff --git a/my-no-sql-core/src/db/db_table/mod.rs b/my-no-sql-core/src/db/db_table/mod.rs index 8c82ee8..419e67b 100644 --- a/my-no-sql-core/src/db/db_table/mod.rs +++ b/my-no-sql-core/src/db/db_table/mod.rs @@ -1,13 +1,13 @@ -mod db_table; #[cfg(feature = "master-node")] mod db_table_attributes; +mod db_table_inner; #[cfg(feature = "master-node")] pub mod db_table_master_node; #[cfg(feature = "master-node")] pub use db_table_attributes::*; -pub use db_table::*; +pub use db_table_inner::*; #[cfg(feature = "master-node")] mod data_to_gc; diff --git a/my-no-sql-sdk/src/lib.rs b/my-no-sql-sdk/src/lib.rs index 1e9206d..f7ec51b 100644 --- a/my-no-sql-sdk/src/lib.rs +++ b/my-no-sql-sdk/src/lib.rs @@ -16,8 +16,5 @@ pub extern crate my_no_sql_tcp_reader as reader; #[cfg(feature = "tcp-contracts")] pub extern crate my_no_sql_tcp_shared as tcp_contracts; -#[cfg(feature = "master-node")] -pub extern crate my_no_sql_server_core as server; - -#[cfg(feature = "read-node")] +#[cfg(any(feature = "master-node", feature = "read-node"))] pub extern crate my_no_sql_server_core as server; diff --git a/my-no-sql-server-core/src/db_instance.rs b/my-no-sql-server-core/src/db_instance.rs index 43ea64f..f3298a5 100644 --- a/my-no-sql-server-core/src/db_instance.rs +++ b/my-no-sql-server-core/src/db_instance.rs @@ -3,10 +3,10 @@ use tokio::sync::RwLock; use std::{collections::HashMap, sync::Arc}; -use super::DbTableWrapper; +use super::DbTable; pub struct DbInstance { - pub tables: RwLock>>, + pub tables: RwLock>>, } impl DbInstance { @@ -26,7 +26,7 @@ impl DbInstance { .collect(); } - pub async fn get_tables(&self) -> Vec> { + pub async fn get_tables(&self) -> Vec> { let read_access = self.tables.read().await; return read_access @@ -36,14 +36,14 @@ impl DbInstance { .collect(); } - pub async fn get_table(&self, table_name: &str) -> Option> { + pub async fn get_table(&self, table_name: &str) -> Option> { let read_access = self.tables.read().await; let result = read_access.get(table_name)?; return Some(result.clone()); } - pub async fn delete_table(&self, table_name: &str) -> Option> { + pub async fn delete_table(&self, table_name: &str) -> Option> { let mut write_access = self.tables.write().await; write_access.remove(table_name) } diff --git a/my-no-sql-server-core/src/db_table_wrapper.rs b/my-no-sql-server-core/src/db_table.rs similarity index 98% rename from my-no-sql-server-core/src/db_table_wrapper.rs rename to my-no-sql-server-core/src/db_table.rs index 5584469..e487f7e 100644 --- a/my-no-sql-server-core/src/db_table_wrapper.rs +++ b/my-no-sql-server-core/src/db_table.rs @@ -8,12 +8,12 @@ use tokio::sync::RwLock; #[cfg(feature = "master-node")] use my_no_sql_core::db::DbTableAttributes; -pub struct DbTableWrapper { +pub struct DbTable { pub name: DbTableName, pub data: RwLock, } -impl DbTableWrapper { +impl DbTable { pub fn new(db_table: DbTableInner) -> Arc { let result = Self { name: db_table.name.clone(), diff --git a/my-no-sql-server-core/src/lib.rs b/my-no-sql-server-core/src/lib.rs index 486b092..20a19fd 100644 --- a/my-no-sql-server-core/src/lib.rs +++ b/my-no-sql-server-core/src/lib.rs @@ -1,7 +1,7 @@ mod db_instance; -mod db_table_wrapper; +mod db_table; pub use db_instance::*; -pub use db_table_wrapper::*; +pub use db_table::*; pub mod db_snapshots; pub extern crate rust_extensions; From 1188088483840995c615fe03d9c9817cfdccfd1d Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 12 Mar 2025 22:32:36 +0100 Subject: [PATCH 80/89] Fixes --- .../src/subscribers/my_no_sql_data_reader_mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs index 7c5182c..c0a72c3 100644 --- a/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs +++ b/my-no-sql-tcp-reader/src/subscribers/my_no_sql_data_reader_mock.rs @@ -75,7 +75,7 @@ where &'s self, partition_key: &'s str, row_key: &'s str, - ) -> GetEntityBuilder { + ) -> GetEntityBuilder<'s, TMyNoSqlEntity> { GetEntityBuilder::new_mock(partition_key, row_key, self.inner.clone()) } From e546b83ebae9000f098adef4da083fd1e2eceeb9 Mon Sep 17 00:00:00 2001 From: amigin Date: Thu, 13 Mar 2025 12:35:20 +0100 Subject: [PATCH 81/89] Added unit tests and fixed bugs in case where we are replace_or_update the same entity with the same Expiration Time --- .../src/db/db_partition/db_rows_container.rs | 69 ++++++++++++++++--- my-no-sql-core/src/expiration_index.rs | 49 +++++++++++-- 2 files changed, 101 insertions(+), 17 deletions(-) diff --git a/my-no-sql-core/src/db/db_partition/db_rows_container.rs b/my-no-sql-core/src/db/db_partition/db_rows_container.rs index bc7a5c6..e0721d8 100644 --- a/my-no-sql-core/src/db/db_partition/db_rows_container.rs +++ b/my-no-sql-core/src/db/db_partition/db_rows_container.rs @@ -69,13 +69,17 @@ impl DbRowsContainer { pub fn insert(&mut self, db_row: Arc) -> Option> { #[cfg(feature = "master-node")] - self.rows_with_expiration_index.add(&db_row); + let added = self.rows_with_expiration_index.add(&db_row); let (_, removed_db_row) = self.data.insert_or_replace(db_row); #[cfg(feature = "master-node")] - if let Some(removed_db_row) = &removed_db_row { - self.rows_with_expiration_index.remove(removed_db_row); + if let Some(added) = added { + if added { + if let Some(removed_db_row) = &removed_db_row { + self.rows_with_expiration_index.remove(removed_db_row); + } + } } removed_db_row @@ -176,7 +180,7 @@ mod expiration_tests { db_rows.insert(Arc::new(db_row)); - assert_eq!(1, db_rows.rows_with_expiration_index.len()) + db_rows.rows_with_expiration_index.assert_len(1); } #[test] @@ -194,7 +198,7 @@ mod expiration_tests { db_rows.insert(Arc::new(db_row)); - assert_eq!(0, db_rows.rows_with_expiration_index.len()) + db_rows.rows_with_expiration_index.assert_len(0); } #[test] @@ -214,7 +218,7 @@ mod expiration_tests { db_rows.remove("test"); - assert_eq!(0, db_rows.rows_with_expiration_index.len()) + db_rows.rows_with_expiration_index.assert_len(0); } #[test] @@ -233,7 +237,7 @@ mod expiration_tests { db_rows.insert(Arc::new(db_row)); - assert_eq!(0, db_rows.rows_with_expiration_index.len()); + db_rows.rows_with_expiration_index.assert_len(0); let new_expiration_time = DateTimeAsMicroseconds::new(2); @@ -245,7 +249,7 @@ mod expiration_tests { .rows_with_expiration_index .has_data_with_expiration_moment(DateTimeAsMicroseconds::new(2)) ); - assert_eq!(1, db_rows.rows_with_expiration_index.len()); + db_rows.rows_with_expiration_index.assert_len(1); } #[test] @@ -272,7 +276,7 @@ mod expiration_tests { .rows_with_expiration_index .has_data_with_expiration_moment(current_expiration) ); - assert_eq!(1, db_rows.rows_with_expiration_index.len()); + db_rows.rows_with_expiration_index.assert_len(1); db_rows.update_expiration_time("test", Some(DateTimeAsMicroseconds::new(2))); @@ -282,7 +286,8 @@ mod expiration_tests { .rows_with_expiration_index .has_data_with_expiration_moment(DateTimeAsMicroseconds::new(2)) ); - assert_eq!(1, db_rows.rows_with_expiration_index.len()); + + db_rows.rows_with_expiration_index.assert_len(1); } #[test] @@ -438,4 +443,48 @@ mod expiration_tests { assert_eq!("test1", db_rows_to_gc.get(0).unwrap().get_row_key()); } + + #[test] + fn check_we_update_row_with_the_same_expiration_date() { + let mut db_rows = DbRowsContainer::new(); + + let row = r#"{"Count":1,"PartitionKey":"in-progress-count1","RowKey":"my-id","Expires":"2025-03-12T10:55:46.0507979Z"}"#; + let now = JsonTimeStamp::now(); + let db_json_entity = DbJsonEntity::parse(row.as_bytes(), &now).unwrap(); + let db_row = Arc::new(db_json_entity.into_db_row().unwrap()); + db_rows.insert(db_row); + db_rows.rows_with_expiration_index.assert_len(1); + + let row = r#"{"Count":1,"PartitionKey":"in-progress-count1","RowKey":"my-id","Expires":"2025-03-12T10:55:46.0507979Z"}"#; + let now = JsonTimeStamp::now(); + let db_json_entity = DbJsonEntity::parse(row.as_bytes(), &now).unwrap(); + let db_row = Arc::new(db_json_entity.into_db_row().unwrap()); + db_rows.insert(db_row); + db_rows.rows_with_expiration_index.assert_len(1); + + db_rows.remove("my-id"); + } + + #[test] + fn check_we_update_same_row_with_new_expiration_date() { + let mut db_rows = DbRowsContainer::new(); + + let row = r#"{"Count":1,"PartitionKey":"in-progress-count1","RowKey":"my-id","Expires":"2025-03-12T10:55:48.0507979Z"}"#; + let now = JsonTimeStamp::now(); + let db_json_entity = DbJsonEntity::parse(row.as_bytes(), &now).unwrap(); + let db_row = Arc::new(db_json_entity.into_db_row().unwrap()); + db_rows.insert(db_row); + db_rows.rows_with_expiration_index.assert_len(1); + + let row = r#"{"Count":1,"PartitionKey":"in-progress-count1","RowKey":"my-id","Expires":"2025-03-12T10:55:50.0507979Z"}"#; + let now = JsonTimeStamp::now(); + let db_json_entity = DbJsonEntity::parse(row.as_bytes(), &now).unwrap(); + let db_row = Arc::new(db_json_entity.into_db_row().unwrap()); + db_rows.insert(db_row); + db_rows.rows_with_expiration_index.assert_len(1); + + db_rows.remove("my-id"); + + db_rows.rows_with_expiration_index.assert_len(0); + } } diff --git a/my-no-sql-core/src/expiration_index.rs b/my-no-sql-core/src/expiration_index.rs index 250ddb9..7c2278a 100644 --- a/my-no-sql-core/src/expiration_index.rs +++ b/my-no-sql-core/src/expiration_index.rs @@ -48,27 +48,43 @@ impl> ExpirationIndexContainer) { + pub fn add(&mut self, item: &impl ExpirationIndex) -> Option { let expiration_moment = item.get_expiration_moment(); if item.get_expiration_moment().is_none() { - return; + return None; } let expiration_moment = expiration_moment.unwrap(); - match self.find_index(expiration_moment) { + let added = match self.find_index(expiration_moment) { Ok(index) => { - self.index[index].items.push(item.to_owned()); + let items = &mut self.index[index].items; + + if items + .iter() + .any(|itm| item.get_id_as_str() == itm.get_id_as_str()) + { + false + } else { + self.index[index].items.push(item.to_owned()); + false + } } Err(index) => { self.index.insert( index, ExpirationIndexItem::new(expiration_moment, item.to_owned()), ); + + true } + }; + + if added { + self.amount += 1; } - self.amount += 1; + Some(added) } pub fn update( @@ -80,7 +96,7 @@ impl> ExpirationIndexContainer) { @@ -111,11 +127,17 @@ impl> ExpirationIndexContainer { - //todo!("Removed - but I have to return it. At least unit test it"); + #[cfg(not(test))] println!( "Somehow we did not find the index for expiration moment {} of '{}'. Expiration moment as rfc3339 is {}", expiration_moment.unix_microseconds, key_as_str, expiration_moment.to_rfc3339() ); + + #[cfg(test)] + panic!( + "Somehow we did not find the index for expiration moment {} of '{}'. Expiration moment as rfc3339 is {}", + expiration_moment.unix_microseconds, key_as_str, expiration_moment.to_rfc3339() + ); } } } @@ -153,6 +175,19 @@ impl> ExpirationIndexContainer Date: Wed, 6 Aug 2025 14:45:46 +0300 Subject: [PATCH 82/89] Updated FlUrl --- .../src/my_no_sql_data_writer/execution.rs | 10 +++++----- my-no-sql-data-writer/src/ping_pool.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 676b7ed..054a98f 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -1,4 +1,4 @@ -use flurl::{FlUrl, FlUrlResponse}; +use flurl::{body::FlUrlBody, FlUrl, FlUrlResponse}; use my_json::{ json_reader::JsonArrayIterator, json_writer::{JsonArrayWriter, RawJsonObject}, @@ -33,7 +33,7 @@ pub async fn create_table_if_not_exists( let fl_url = params.populate_params(fl_url); - let mut response = fl_url.post(None).await?; + let mut response = fl_url.post(FlUrlBody::Empty).await?; create_table_errors_handler(&mut response, "create_table_if_not_exists", url).await } @@ -53,7 +53,7 @@ pub async fn create_table( let fl_url = params.populate_params(fl_url); - let mut response = fl_url.post(None).await?; + let mut response = fl_url.post(FlUrlBody::Empty).await?; create_table_errors_handler(&mut response, "create_table", url).await } @@ -68,7 +68,7 @@ pub async fn insert_entity Date: Wed, 6 Aug 2025 15:16:33 +0300 Subject: [PATCH 83/89] Fixes --- .../src/my_no_sql_data_writer/execution.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 054a98f..615ae4b 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -483,9 +483,9 @@ fn is_ok_result(response: &FlUrlResponse) -> bool { fn serialize_entities_to_body( entities: &[TEntity], -) -> Option> { +) -> FlUrlBody { if entities.len() == 0 { - return Some(vec![b'[', b']']); + FlUrlBody::new_as_json(vec![b'[', b']']); } let mut json_array_writer = JsonArrayWriter::new(); @@ -496,7 +496,7 @@ fn serialize_entities_to_body( json_array_writer.write(payload); } - Some(json_array_writer.build().into_bytes()) + FlUrlBody::new_as_json(json_array_writer.build().into_bytes()) } async fn check_error(response: &mut FlUrlResponse) -> Result<(), DataWriterError> { @@ -678,8 +678,8 @@ mod tests { }, ]; - let as_json = super::serialize_entities_to_body(&entities).unwrap(); + let as_json = super::serialize_entities_to_body(&entities); - println!("{}", std::str::from_utf8(&as_json).unwrap()); + println!("{}", std::str::from_utf8(as_json.as_slice()).unwrap()); } } From bd02edb83c7fa8e282bbbad1f14b87463ded8b40 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 6 Aug 2025 15:53:33 +0300 Subject: [PATCH 84/89] Debugging bulk_insert_or_replace --- .../src/my_no_sql_data_writer/execution.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 615ae4b..30d6f48 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -118,14 +118,22 @@ pub async fn bulk_insert_or_replace< return Ok(()); } + let mut debug_data = String::new(); + let response = flurl .append_path_segment(BULK_CONTROLLER) .append_path_segment("InsertOrReplace") .append_data_sync_period(sync_period) .with_table_name_as_query_param(TEntity::TABLE_NAME) - .post(serialize_entities_to_body(entities)) + .post_with_debug(serialize_entities_to_body(entities), &mut debug_data) .await?; + if debug_data.len() > 1024 { + println!("Debug Data: {}", &debug_data[..1024]); + } else { + println!("Debug Data: {}", debug_data); + } + if is_ok_result(&response) { return Ok(()); } From 306fdd959aa418f4912cad370afad53c740d9d64 Mon Sep 17 00:00:00 2001 From: amigin Date: Wed, 6 Aug 2025 16:36:34 +0300 Subject: [PATCH 85/89] Removed Debug Data --- .../src/my_no_sql_data_writer/execution.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 30d6f48..615ae4b 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -118,22 +118,14 @@ pub async fn bulk_insert_or_replace< return Ok(()); } - let mut debug_data = String::new(); - let response = flurl .append_path_segment(BULK_CONTROLLER) .append_path_segment("InsertOrReplace") .append_data_sync_period(sync_period) .with_table_name_as_query_param(TEntity::TABLE_NAME) - .post_with_debug(serialize_entities_to_body(entities), &mut debug_data) + .post(serialize_entities_to_body(entities)) .await?; - if debug_data.len() > 1024 { - println!("Debug Data: {}", &debug_data[..1024]); - } else { - println!("Debug Data: {}", debug_data); - } - if is_ok_result(&response) { return Ok(()); } From d012ea61fd17978b3a70812928731cfb59526eb7 Mon Sep 17 00:00:00 2001 From: amigin Date: Sat, 16 Aug 2025 14:53:15 +0300 Subject: [PATCH 86/89] Fixes --- my-no-sql-core/src/db/db_partition/db_partition.rs | 2 +- my-no-sql-core/src/db/db_partition/db_rows_container.rs | 2 +- my-no-sql-core/src/db/db_table/db_partitions_container.rs | 4 ++-- my-no-sql-core/src/db/db_table/db_table_inner.rs | 2 +- .../src/my_no_sql_data_writer/execution.rs | 8 ++++---- my-no-sql-data-writer/src/ping_pool.rs | 3 ++- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/my-no-sql-core/src/db/db_partition/db_partition.rs b/my-no-sql-core/src/db/db_partition/db_partition.rs index 0d15cb1..5ab014c 100644 --- a/my-no-sql-core/src/db/db_partition/db_partition.rs +++ b/my-no-sql-core/src/db/db_partition/db_partition.rs @@ -132,7 +132,7 @@ impl DbPartition { result.get_result() } - pub fn get_all_rows(&self) -> std::slice::Iter> { + pub fn get_all_rows<'s>(&'s self) -> std::slice::Iter<'s, Arc> { self.rows.get_all() } diff --git a/my-no-sql-core/src/db/db_partition/db_rows_container.rs b/my-no-sql-core/src/db/db_partition/db_rows_container.rs index e0721d8..09fdb81 100644 --- a/my-no-sql-core/src/db/db_partition/db_rows_container.rs +++ b/my-no-sql-core/src/db/db_partition/db_rows_container.rs @@ -104,7 +104,7 @@ impl DbRowsContainer { return self.data.contains(row_key); } - pub fn get_all(&self) -> std::slice::Iter> { + pub fn get_all<'s>(&'s self) -> std::slice::Iter<'s, Arc> { self.data.iter() } diff --git a/my-no-sql-core/src/db/db_table/db_partitions_container.rs b/my-no-sql-core/src/db/db_table/db_partitions_container.rs index d093b59..ffc33c1 100644 --- a/my-no-sql-core/src/db/db_table/db_partitions_container.rs +++ b/my-no-sql-core/src/db/db_table/db_partitions_container.rs @@ -32,11 +32,11 @@ impl DbPartitionsContainer { self.partitions.len() } - pub fn get_partitions(&self) -> std::slice::Iter { + pub fn get_partitions<'s>(&'s self) -> std::slice::Iter<'s, DbPartition> { self.partitions.iter() } - pub fn get_partitions_mut(&mut self) -> std::slice::IterMut { + pub fn get_partitions_mut<'s>(&'s mut self) -> std::slice::IterMut<'s, DbPartition> { self.partitions.iter_mut() } #[cfg(feature = "master-node")] diff --git a/my-no-sql-core/src/db/db_table/db_table_inner.rs b/my-no-sql-core/src/db/db_table/db_table_inner.rs index 52459ef..67d0b2a 100644 --- a/my-no-sql-core/src/db/db_table/db_table_inner.rs +++ b/my-no-sql-core/src/db/db_table/db_table_inner.rs @@ -117,7 +117,7 @@ impl DbTableInner { self.partitions.get(partition_key) } #[inline] - pub fn get_partitions(&self) -> std::slice::Iter { + pub fn get_partitions<'s>(&'s self) -> std::slice::Iter<'s, DbPartition> { self.partitions.get_partitions() } } diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 615ae4b..588b0fa 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -68,7 +68,7 @@ pub async fn insert_entity( entities: &[TEntity], ) -> FlUrlBody { if entities.len() == 0 { - FlUrlBody::new_as_json(vec![b'[', b']']); + FlUrlBody::Json(vec![b'[', b']']); } let mut json_array_writer = JsonArrayWriter::new(); @@ -496,7 +496,7 @@ fn serialize_entities_to_body( json_array_writer.write(payload); } - FlUrlBody::new_as_json(json_array_writer.build().into_bytes()) + FlUrlBody::Json(json_array_writer.build().into_bytes()) } async fn check_error(response: &mut FlUrlResponse) -> Result<(), DataWriterError> { diff --git a/my-no-sql-data-writer/src/ping_pool.rs b/my-no-sql-data-writer/src/ping_pool.rs index 75478b8..275a4e0 100644 --- a/my-no-sql-data-writer/src/ping_pool.rs +++ b/my-no-sql-data-writer/src/ping_pool.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; +use flurl::body::FlUrlBody; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; @@ -111,7 +112,7 @@ async fn ping_loop() { .with_retries(3) .append_path_segment("api") .append_path_segment("ping") - .post(&ping_model) + .post(FlUrlBody::as_json(&ping_model)) .await; if let Err(err) = &fl_url_response { From 0674964a9d8d1662d6a72a576d5ceb6318fd6ba6 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 5 Sep 2025 11:08:59 +0300 Subject: [PATCH 87/89] my-json: 0.3.2 --- my-no-sql-core/Cargo.toml | 2 +- my-no-sql-core/src/db/db_partition/db_partition.rs | 9 +++++---- my-no-sql-core/src/db/db_row/db_row.rs | 8 +++++--- my-no-sql-core/src/db/db_table/db_table_inner.rs | 4 ++-- my-no-sql-data-writer/Cargo.toml | 2 +- .../src/my_no_sql_data_writer/execution.rs | 2 +- .../src/db_snapshots/db_rows_by_partitions_snapshot.rs | 2 +- .../src/db_snapshots/db_rows_snapshot.rs | 2 +- .../src/db_snapshots/db_table_snapshot.rs | 2 +- my-no-sql-tcp-reader/Cargo.toml | 2 +- 10 files changed, 19 insertions(+), 16 deletions(-) diff --git a/my-no-sql-core/Cargo.toml b/my-no-sql-core/Cargo.toml index 4e9e9be..b2edce4 100644 --- a/my-no-sql-core/Cargo.toml +++ b/my-no-sql-core/Cargo.toml @@ -13,7 +13,7 @@ debug_db_row = [] [dependencies] my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } -my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.2", git = "https://github.com/MyJetTools/my-json.git" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } serde_json = { version = "*" } diff --git a/my-no-sql-core/src/db/db_partition/db_partition.rs b/my-no-sql-core/src/db/db_partition/db_partition.rs index 5ab014c..32952b8 100644 --- a/my-no-sql-core/src/db/db_partition/db_partition.rs +++ b/my-no-sql-core/src/db/db_partition/db_partition.rs @@ -1,4 +1,4 @@ -use my_json::json_writer::JsonObject; +use my_json::json_writer::JsonValueWriter; #[cfg(feature = "master-node")] use rust_extensions::date_time::AtomicDateTimeAsMicroseconds; @@ -200,8 +200,9 @@ impl DbPartition { } } -impl JsonObject for &'_ DbPartition { - fn write_into(&self, dest: &mut String) { +impl JsonValueWriter for &'_ DbPartition { + const IS_ARRAY: bool = false; + fn write(&self, dest: &mut String) { let mut first_element = true; dest.push('['); for db_row in self.rows.get_all() { @@ -210,7 +211,7 @@ impl JsonObject for &'_ DbPartition { } else { dest.push(','); } - db_row.as_ref().write_into(dest) + db_row.as_ref().write(dest) } dest.push(']'); } diff --git a/my-no-sql-core/src/db/db_row/db_row.rs b/my-no-sql-core/src/db/db_row/db_row.rs index 1f4a83a..d4572ee 100644 --- a/my-no-sql-core/src/db/db_row/db_row.rs +++ b/my-no-sql-core/src/db/db_row/db_row.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use my_json::json_writer::JsonObject; +use my_json::json_writer::JsonValueWriter; #[cfg(feature = "master-node")] use rust_extensions::date_time::AtomicDateTimeAsMicroseconds; #[cfg(feature = "master-node")] @@ -269,8 +269,10 @@ fn find_json_separator_after(src: &[u8], pos: usize) -> Option { None } -impl JsonObject for &'_ DbRow { - fn write_into(&self, dest: &mut String) { +impl JsonValueWriter for &'_ DbRow { + const IS_ARRAY: bool = false; + + fn write(&self, dest: &mut String) { self.write_json(dest) } } diff --git a/my-no-sql-core/src/db/db_table/db_table_inner.rs b/my-no-sql-core/src/db/db_table/db_table_inner.rs index 67d0b2a..9af04c6 100644 --- a/my-no-sql-core/src/db/db_table/db_table_inner.rs +++ b/my-no-sql-core/src/db/db_table/db_table_inner.rs @@ -71,7 +71,7 @@ impl DbTableInner { for db_partition in self.partitions.get_partitions() { for db_row in db_partition.get_all_rows() { - json_array_writer.write(db_row.as_ref()) + json_array_writer = json_array_writer.write(db_row.as_ref()); } } @@ -100,7 +100,7 @@ impl DbTableInner { if let Some(db_partition) = self.partitions.get(partition_key) { for db_row in db_partition.get_all_rows() { - json_array_writer.write(db_row.as_ref()) + json_array_writer = json_array_writer.write(db_row.as_ref()); } } diff --git a/my-no-sql-data-writer/Cargo.toml b/my-no-sql-data-writer/Cargo.toml index 8ccb1d4..c6a6365 100644 --- a/my-no-sql-data-writer/Cargo.toml +++ b/my-no-sql-data-writer/Cargo.toml @@ -19,7 +19,7 @@ rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-ext ] } flurl = { tag = "0.6.1", git = "https://github.com/MyJetTools/fl-url" } -my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.2", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 588b0fa..9085c21 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -493,7 +493,7 @@ fn serialize_entities_to_body( for entity in entities { let payload = entity.serialize_entity(); let payload: RawJsonObject = payload.into(); - json_array_writer.write(payload); + json_array_writer = json_array_writer.write(payload); } FlUrlBody::Json(json_array_writer.build().into_bytes()) diff --git a/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs index dec7bb8..89d8539 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_rows_by_partitions_snapshot.rs @@ -52,7 +52,7 @@ impl DbRowsByPartitionsSnapshot { let mut json_array_writer = JsonArrayWriter::new(); for (_, snapshot) in self.partitions.iter() { for db_row in snapshot { - json_array_writer.write(db_row.as_ref()); + json_array_writer = json_array_writer.write(db_row.as_ref()); } } diff --git a/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs index 5f22325..28288e5 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_rows_snapshot.rs @@ -35,7 +35,7 @@ impl DbRowsSnapshot { pub fn as_json_array(&self) -> JsonArrayWriter { let mut json_array_writer = JsonArrayWriter::new(); for db_row in &self.db_rows { - json_array_writer.write(db_row.as_ref()); + json_array_writer = json_array_writer.write(db_row.as_ref()); } json_array_writer diff --git a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs index b245771..6267700 100644 --- a/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs +++ b/my-no-sql-server-core/src/db_snapshots/db_table_snapshot.rs @@ -38,7 +38,7 @@ impl DbTableSnapshot { for db_partition_snapshot in self.by_partition.iter() { for db_row in &db_partition_snapshot.db_rows_snapshot.db_rows { - json_array_writer.write(db_row.as_ref()); + json_array_writer = json_array_writer.write(db_row.as_ref()); } } diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index 9af20d3..b5c0a7e 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -16,7 +16,7 @@ my-no-sql-core = { path = "../my-no-sql-core" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } my-tcp-sockets = { tag = "0.1.11", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } my-logger = { tag = "1.2.1", git = "https://github.com/MyJetTools/my-logger.git" } -my-json = { tag = "0.3.1", git = "https://github.com/MyJetTools/my-json.git" } +my-json = { tag = "0.3.2", git = "https://github.com/MyJetTools/my-json.git" } tokio = { version = "*", features = ["full"] } tokio-util = "*" From 861709f6d8e02cf32f4ad5c617c9337496f1f1c8 Mon Sep 17 00:00:00 2001 From: amigin Date: Mon, 24 Nov 2025 14:33:23 +0200 Subject: [PATCH 88/89] Unix-socket support enabled --- my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs | 4 +++- my-no-sql-data-writer/src/ping_pool.rs | 2 +- my-no-sql-tcp-reader/Cargo.toml | 2 +- my-no-sql-tcp-shared/Cargo.toml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs index 9085c21..c1a06d9 100644 --- a/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs +++ b/my-no-sql-data-writer/src/my_no_sql_data_writer/execution.rs @@ -680,6 +680,8 @@ mod tests { let as_json = super::serialize_entities_to_body(&entities); - println!("{}", std::str::from_utf8(as_json.as_slice()).unwrap()); + let body = as_json.into_vec(); + + println!("{}", std::str::from_utf8(&body).unwrap()); } } diff --git a/my-no-sql-data-writer/src/ping_pool.rs b/my-no-sql-data-writer/src/ping_pool.rs index 275a4e0..a5ca77a 100644 --- a/my-no-sql-data-writer/src/ping_pool.rs +++ b/my-no-sql-data-writer/src/ping_pool.rs @@ -86,7 +86,7 @@ async fn ping_loop() { let url = settings.get_url().await; let entry = url_to_ping .entry(url) - .or_insert_with(|| ((settings.clone(), Vec::new()))); + .or_insert_with(|| (settings.clone(), Vec::new())); entry.1.push(table.to_string()); } diff --git a/my-no-sql-tcp-reader/Cargo.toml b/my-no-sql-tcp-reader/Cargo.toml index b5c0a7e..504a71d 100644 --- a/my-no-sql-tcp-reader/Cargo.toml +++ b/my-no-sql-tcp-reader/Cargo.toml @@ -14,7 +14,7 @@ my-no-sql-abstractions = { path = "../my-no-sql-abstractions" } my-no-sql-core = { path = "../my-no-sql-core" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } -my-tcp-sockets = { tag = "0.1.11", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } +my-tcp-sockets = { tag = "0.1.12", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } my-logger = { tag = "1.2.1", git = "https://github.com/MyJetTools/my-logger.git" } my-json = { tag = "0.3.2", git = "https://github.com/MyJetTools/my-json.git" } diff --git a/my-no-sql-tcp-shared/Cargo.toml b/my-no-sql-tcp-shared/Cargo.toml index 0c4eb28..7fad6da 100644 --- a/my-no-sql-tcp-shared/Cargo.toml +++ b/my-no-sql-tcp-shared/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -my-tcp-sockets = { tag = "0.1.11", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } +my-tcp-sockets = { tag = "0.1.12", git = "https://github.com/MyJetTools/my-tcp-sockets.git" } rust-extensions = { tag = "0.1.5", git = "https://github.com/MyJetTools/rust-extensions.git" } tokio = { version = "*", features = ["full"] } From 2921affa655062066793af4cbb8cbf84459ae5f7 Mon Sep 17 00:00:00 2001 From: amigin Date: Tue, 13 Jan 2026 12:58:50 +0200 Subject: [PATCH 89/89] Added Last Updates --- MY_NO_SQL_ENTITY_DESIGN_PATTERNS.md | 278 ++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 MY_NO_SQL_ENTITY_DESIGN_PATTERNS.md diff --git a/MY_NO_SQL_ENTITY_DESIGN_PATTERNS.md b/MY_NO_SQL_ENTITY_DESIGN_PATTERNS.md new file mode 100644 index 0000000..ed0c221 --- /dev/null +++ b/MY_NO_SQL_ENTITY_DESIGN_PATTERNS.md @@ -0,0 +1,278 @@ +## MyNoSql Entity Design Patterns + +### Purpose +- Provide consistent guidance for defining MyNoSql entities and enum-based models used by readers/writers. +- Align with MyNoSql server concepts of `PartitionKey`, `RowKey`, `TimeStamp`, and optional `Expires`. +- Serve as an instruction set for AI agents so generated code follows established patterns without guesswork. + +### Core principles +- Entities always carry partition, row, and timestamp metadata; these identify rows per the server model. +- Use `my-no-sql-macros` to autogenerate required fields and trait impls instead of hand-writing boilerplate. +- Serde names `PartitionKey`, `RowKey`, `TimeStamp`, and `Expires` are reserved; avoid collisions or custom renames that reuse them. +- Table rows are identified by `(PartitionKey, RowKey)`; partitions group related rows; `TimeStamp` tracks server-side update time. +- Writers use HTTP CRUD; readers subscribe and keep local copies—design entities to be compact and stable. + +### Table name validation (my-no-sql-server) +- Length: 3–63 characters. +- Allowed characters: lowercase latin letters `a`-`z`, digits `0`-`9`, and dash `-`. +- No consecutive dashes (e.g., `my--table-name` is invalid). +- Do not start or end with a dash; prefer starting with a letter (e.g., `-my-table-name` and `my-table-name-` are invalid). +- Keep names lowercase and stable; changing table names is a breaking change for both readers and writers. +- Reference: [Table Name Validation](https://github.com/MyJetTools/my-no-sql-server/wiki/Table-Name-Validation). + +### Where entities live and how they flow +- Entities are stored in `my-no-sql-server`. +- Reader API: `MyNoSqlDataReader` subscribes to the table and keeps the latest snapshot locally in the application for fast reads. +- Writer API: `MyNoSqlDataWriter` is used by writers to push inserts/updates/deletes to the server. +- Design entities so they are stable over time; breaking schema changes can break both readers (subscriptions) and writers (HTTP). + +### Pattern 1: Table = Model +- Use `#[my_no_sql_entity("table-name")]` on a struct that represents a single table’s rows. +- The macro injects: + - `partition_key: String`, `row_key: String`, `time_stamp: Timestamp` with proper Serde casing. + - `MyNoSqlEntity` and `MyNoSqlEntitySerializer` impls (serialize/deserialize ready for readers/writers). +- Optional `with_expires = true` adds `expires: Timestamp` with Serde name `Expires` for TTL semantics. +- Derive serde and clone/debug as needed; apply `#[serde(rename_all = "...")]` for payload fields to keep consistent casing. +- Example: + ```rust + #[my_no_sql_macros::my_no_sql_entity("bidask-snapshots")] + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct BidAskSnapshot { + pub unix_timestamp_with_milis: u64, + pub bid: f64, + pub ask: f64, + pub base: String, + pub quote: String, + } + ``` +- Example with TTL: + ```rust + #[my_no_sql_macros::my_no_sql_entity("sessions", with_expires = true)] + #[derive(Serialize, Deserialize, Clone)] + pub struct SessionEntity { + pub user_id: String, + pub token: String, + pub device: String, + // `expires` is auto-added by the macro because with_expires = true + } + ``` + +### Pattern 2: PartitionKey + RowKey = Model (enum) +- Use when each `(PartitionKey, RowKey)` pair maps to a distinct model type. +- Declare an enum with `#[enum_of_my_no_sql_entity(table_name:"", generate_unwraps)]`. +- For each variant model: + - Annotate with `#[enum_model(partition_key:"...", row_key:"...")]`. + - Derive serde and other traits required by your service. +- The macros generate: + - Correct partition/row keys per variant. + - Serialization helpers plus `unwrap_caseX` accessors when `generate_unwraps` is set. +- Example: + ```rust + #[enum_of_my_no_sql_entity(table_name:"Test", generate_unwraps)] + pub enum MyNoSqlEnumEntityTest { + Case1(Struct1), + Case2(Struct2), + } + + #[enum_model(partition_key:"pk1", row_key:"rk1")] + #[derive(Serialize, Deserialize, Clone)] + pub struct Struct1 { + pub field1: String, + pub field2: i32, + } + + #[enum_model(partition_key:"pk2", row_key:"rk2")] + #[derive(Serialize, Deserialize, Clone)] + pub struct Struct2 { + pub field3: String, + pub field4: i32, + } + ``` +- More elaborate enum example (mixed casing and TTL in one variant): + ```rust + #[enum_of_my_no_sql_entity(table_name:"notifications", generate_unwraps)] + pub enum NotificationEntity { + #[enum_model(partition_key:"email", row_key:"welcome")] + EmailWelcome(EmailWelcomeModel), + #[enum_model(partition_key:"push", row_key:"security")] + PushSecurity(PushSecurityModel), + } + + #[derive(Serialize, Deserialize, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct EmailWelcomeModel { + pub subject: String, + pub body: String, + } + + #[derive(Serialize, Deserialize, Clone)] + #[serde(rename_all = "camelCase")] + pub struct PushSecurityModel { + pub title: String, + pub message: String, + pub severity: String, + } + ``` + +### Field and serde rules +- Do not rename payload fields to reserved serde names (`PartitionKey`, `RowKey`, `TimeStamp`, `Expires`); the macro enforces uniqueness. +- When adding an `expires` field manually, use type `Timestamp`; otherwise enable `with_expires`. +- Keep `LAZY_DESERIALIZATION` as the default (`false`) unless the macro adds support for a specialized flow. +- Prefer `#[serde(rename_all = "...")]` instead of renaming individual fields when possible. +- Avoid floats for keys; keep keys ASCII-safe and stable (no trailing slashes or whitespace). + +### Testing guidance +- Validate serialization/deserialization paths using the examples under `my-no-sql-tests/src/macros_tests`. +- For enums, assert `unwrap_caseX` helpers and key selection per variant. +- Add unit tests per model that: + - Serialize then deserialize and compare fields. + - For enums, confirm correct variant after round-trip and that unwrap helpers work. + - Validate `expires` presence when `with_expires = true`. + +### When to choose each pattern +- Use **Table = Model** when all rows in a table share the same schema. +- Use **PartitionKey + RowKey = Model (enum)** when a table hosts heterogeneous payloads selected by keys. + +### Key design guidance +- PartitionKey: + - Group data that is frequently read together. + - Keep length modest; avoid unbounded cardinality when possible. +- RowKey: + - Unique within a partition. + - Prefer stable identifiers (ids, symbols, timestamps encoded consistently). +- TimeStamp: + - Auto-managed; used by server for last-write tracking and sync ordering. +- Expires (TTL): + - Use for session-like or cache-like data; choose UTC epoch milliseconds. + +### Best practices: constant keys +- When PartitionKey and/or RowKey are fixed for a table, expose them as `const` on the entity and initialize defaults with those constants. This keeps writers/readers aligned and avoids typos. +- Example: + ```rust + use serde::*; + + service_sdk::macros::use_my_no_sql_entity!(); + + #[my_no_sql_entity("atr-settings")] + #[derive(Serialize, Deserialize, Debug, Clone)] + pub struct AtrSettingsEntity { + pub percent: f64, + pub candles_count: i32, + } + + impl AtrSettingsEntity { + pub const PARTITION_KEY: &'static str = "*"; + pub const ROW_KEY: &'static str = "*"; + } + + impl Default for AtrSettingsEntity { + fn default() -> Self { + Self { + partition_key: Self::PARTITION_KEY.to_string(), + row_key: Self::ROW_KEY.to_string(), + time_stamp: Default::default(), + percent: 0.8, + candles_count: 365, + } + } + } + ``` + +### Best practices: meaningful keys as helpers +- When keys encode business meaning, provide helper functions to generate and to read them back, to avoid duplicating string literals across the codebase. +- Example: + ```rust + use serde::*; + use trading_robot_common::CandleType; + + service_sdk::macros::use_my_no_sql_entity!(); + + // PartitionKey: instrument_id + // RowKey: interval ("1m", "5m", "1h", "1d", "1M") + #[my_no_sql_entity("atr-values")] + #[derive(Serialize, Deserialize, Debug, Clone)] + pub struct AtrValueMyNoSqlEntity { + pub value: f64, + } + + impl AtrValueMyNoSqlEntity { + pub fn generate_partition_key(instrument_id: &str) -> &str { + instrument_id + } + + pub fn generate_row_key(candle_type: CandleType) -> &'static str { + candle_type.as_str() + } + + pub fn get_instrument(&self) -> &str { + &self.partition_key + } + + pub fn get_candle_type(&self) -> CandleType { + CandleType::from_str(&self.row_key) + } + } + ``` + +### Macro options cheat sheet +- `#[my_no_sql_entity("table", with_expires = true)]` → adds `expires`. +- `#[enum_of_my_no_sql_entity(table_name:"table", generate_unwraps)]` → generates unwrap helpers per variant. +- `#[enum_model(partition_key:"...", row_key:"...")]` → binds a variant model to fixed keys. +- The macros auto-implement: + - `MyNoSqlEntity` with `TABLE_NAME`, `LAZY_DESERIALIZATION = false`, `get_partition_key`, `get_row_key`, `get_time_stamp`. + - `MyNoSqlEntitySerializer` with standard binary serialize/deserialize. + +### Minimal working template (table = model) +```rust +use my_no_sql_macros::my_no_sql_entity; +use serde::{Deserialize, Serialize}; + +#[my_no_sql_entity("table-name")] +#[derive(Serialize, Deserialize, Clone)] +pub struct MyEntity { + pub value: String, + pub count: i32, +} +``` + +### Minimal working template (enum model) +```rust +use my_no_sql_macros::{enum_model, enum_of_my_no_sql_entity}; +use serde::{Deserialize, Serialize}; + +#[enum_of_my_no_sql_entity(table_name:"events", generate_unwraps)] +pub enum EventEntity { + Create(CreateEvent), + Delete(DeleteEvent), +} + +#[enum_model(partition_key:"pk-create", row_key:"rk-create")] +#[derive(Serialize, Deserialize, Clone)] +pub struct CreateEvent { + pub id: String, + pub payload: String, +} + +#[enum_model(partition_key:"pk-delete", row_key:"rk-delete")] +#[derive(Serialize, Deserialize, Clone)] +pub struct DeleteEvent { + pub id: String, + pub reason: String, +} +``` + +### AI generation checklist +- Choose the pattern (single model vs enum). +- Set `table_name` accurately and keep it stable. +- Apply `#[serde(rename_all = "...")]` for payload fields. +- Never introduce fields named `PartitionKey`, `RowKey`, `TimeStamp`, `Expires` manually unless following macro requirements. +- For TTL needs, prefer `with_expires = true` over a custom `expires` unless special handling is required. +- Derive `Serialize`, `Deserialize`, and `Clone`; add `Debug` when useful for logs/tests. +- Provide unit tests that round-trip entities and assert variant unwraps. + +### Anti-patterns to avoid +- Manually defining partition/row/timestamp fields when using the macros (causes duplication or serde conflicts). +- Reusing the same `(partition_key, row_key)` for multiple variants in enum models. +- Using dynamic keys in enum models (keys must be compile-time constants in attributes). +- Introducing serde renames that clash with reserved casing tokens.