diff --git a/Cargo.lock b/Cargo.lock index ad8e68d1f9..30fb62d398 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3000,9 +3000,7 @@ dependencies = [ [[package]] name = "snarkvm" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f504949c916d4e2d7cb3f7a2623b4087f9be9de36d9a6d3d28b99980315959" +version = "0.14.5" dependencies = [ "anstyle", "anyhow", @@ -3030,9 +3028,7 @@ dependencies = [ [[package]] name = "snarkvm-algorithms" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d777e8f8ef2fb0e9e685f4d1825d6d2dbaac4506e5c686aa927674a0cafea265" +version = "0.14.5" dependencies = [ "aleo-std", "anyhow", @@ -3060,9 +3056,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8519ce9900226b42ce4c8c84aac66f03ae38f52cbcc5d9522344e5376d337e8" +version = "0.14.5" dependencies = [ "snarkvm-circuit-account", "snarkvm-circuit-algorithms", @@ -3075,9 +3069,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-account" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9a6bf8a922c3948415eaac249691739ea7ad901360abeb395b196b03d06b58" +version = "0.14.5" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-network", @@ -3087,9 +3079,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-algorithms" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78f858a3af5343eb48037323ac64a4a186f998cf4dd3c05417045ac64fa0ad4" +version = "0.14.5" dependencies = [ "snarkvm-circuit-types", "snarkvm-console-algorithms", @@ -3098,9 +3088,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-collections" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c070a5841440368194e8e4fb5e7f6be508c5003970a2a19a7c90bbca4923dcf" +version = "0.14.5" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-types", @@ -3109,9 +3097,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-environment" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8b388c125b85e80a6ecaf58cc7573752dfc385e2324cc48a99ca9486ba7752f" +version = "0.14.5" dependencies = [ "indexmap 2.0.0", "itertools", @@ -3128,15 +3114,11 @@ dependencies = [ [[package]] name = "snarkvm-circuit-environment-witness" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f70416fbb428c57393bb987159bbbe5257fa6971785e7b44240bd9979b584b7" +version = "0.14.5" [[package]] name = "snarkvm-circuit-network" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4d6c74287202a157257568038a6faf601ae79b10dd49769d43ee8a218fe6511" +version = "0.14.5" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-collections", @@ -3146,9 +3128,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-program" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23cf4c4a3dcf22b621de6f65d7ed2772a1ced8ef8a3be0c963bf62d559f15e1" +version = "0.14.5" dependencies = [ "snarkvm-circuit-account", "snarkvm-circuit-collections", @@ -3160,9 +3140,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eedbdb23e4721e38c5241f83472b4d593f1c06c05a0e962bb2cff64b89a085cc" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-address", @@ -3176,9 +3154,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-address" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3a71dcc2902670e4ad57486a97fbddc3b51a0edda79ff5e9123c3a4780eff5" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -3190,9 +3166,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-boolean" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d39043946b6f9b1d7c7be63525f813aba15e773b24772892d5d01ef7c6fe815" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-console-types-boolean", @@ -3200,9 +3174,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-field" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545b7d12c0af5826723727b176180e913d662d6d8edf4481edafe22fcf61d0b4" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -3211,9 +3183,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-group" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "315d6d9e4bcfa33d933c7c28387094817024f977026144134ffe76197130a230" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -3224,9 +3194,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-integers" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5aa11ffca4e9f1c07fcd0c4120ca19d59a5e17446e318b383b11f64da56068" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -3236,9 +3204,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-scalar" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42fd8e73966d9af78587c22f874532b49c2c17727cb4ec99ed330a5a199e7a3" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -3248,9 +3214,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-string" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35150cac0a7a43d59a996f020b6996a9aa94313971bb5bced7aceac8263a6c3b" +version = "0.14.5" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -3261,9 +3225,7 @@ dependencies = [ [[package]] name = "snarkvm-console" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1753947bbf7c7f903010ca575146629908e55cf9a0be80d12cca9dbc5955329e" +version = "0.14.5" dependencies = [ "snarkvm-console-account", "snarkvm-console-algorithms", @@ -3275,9 +3237,7 @@ dependencies = [ [[package]] name = "snarkvm-console-account" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b56bb693e83dd45ac2faa81d31f914aa9ce5b7bcc7f779720b68ac0cb6f3e5c" +version = "0.14.5" dependencies = [ "bs58", "snarkvm-console-network", @@ -3286,9 +3246,7 @@ dependencies = [ [[package]] name = "snarkvm-console-algorithms" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617d4c42089f47e0ad14e93492800d4b70e9013f65cd2a4782d7e6384c89fedc" +version = "0.14.5" dependencies = [ "blake2s_simd", "smallvec", @@ -3299,9 +3257,7 @@ dependencies = [ [[package]] name = "snarkvm-console-collections" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffaf08fdfc1b4bcbc0c1742232515052197ee81370f27f340647f4241abe3f7" +version = "0.14.5" dependencies = [ "aleo-std", "rayon", @@ -3311,9 +3267,7 @@ dependencies = [ [[package]] name = "snarkvm-console-network" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebbf5866302e2ef5ddefded5bff934510a1459d11c9a67b60a3a6b3d960ff94b" +version = "0.14.5" dependencies = [ "anyhow", "indexmap 2.0.0", @@ -3335,9 +3289,7 @@ dependencies = [ [[package]] name = "snarkvm-console-network-environment" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464f90646767eda24094f5b82424842bb43f12c5bd0b12fa333725b5242692a2" +version = "0.14.5" dependencies = [ "anyhow", "bech32", @@ -3353,9 +3305,7 @@ dependencies = [ [[package]] name = "snarkvm-console-program" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "315c9a2048a4360f0248513abb2645f110d50ae342e9e5e84126c1e24e2d1a1b" +version = "0.14.5" dependencies = [ "enum_index", "enum_index_derive", @@ -3373,9 +3323,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "def389ab6f72e1065f98a202a17f5a227a4a289081e86a1c6bd01201a215156c" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-address", @@ -3389,9 +3337,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-address" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d618665c79ef0f7b29f43d094ca583443a7c3e53d3b03124ae240e24888612b" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -3401,18 +3347,14 @@ dependencies = [ [[package]] name = "snarkvm-console-types-boolean" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b1dc791f836634b3da873d988c760ffacf467fa842b45e1e09255d2e7e8144" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", ] [[package]] name = "snarkvm-console-types-field" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5d771435d7f9584560986cfef2793f4973b46e818861575f482c73a7d564f0" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -3420,9 +3362,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-group" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50004e5cd9c87178d8cd292b8990508979ccd391c232d2b7a6c8db2aa8bd53bd" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -3432,9 +3372,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-integers" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cfc7f40aa8bb0c85e1979e198ecfec74c99814ee198daeeace0beee87e3ba07" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -3443,9 +3381,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-scalar" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5347cab95a75bef3405f635d810411f5da7c9ec054d840abc0cadedd886a05a" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -3454,9 +3390,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-string" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968ce0486837a721b723375a42ce1011495b60c376b4a5bb14637155be82006a" +version = "0.14.5" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -3466,9 +3400,7 @@ dependencies = [ [[package]] name = "snarkvm-curves" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40e75e3ede57bd082e123958bfa6ecad1dd38e208141e0e285bc4995dcd06ee5" +version = "0.14.5" dependencies = [ "rand", "rayon", @@ -3481,9 +3413,7 @@ dependencies = [ [[package]] name = "snarkvm-fields" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4e9d29b38e0a0d69740c5df23b838fa5df7da5e5781e3f07bedbc480b34b2d" +version = "0.14.5" dependencies = [ "aleo-std", "anyhow", @@ -3499,9 +3429,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fb2d805d561826e4490e403138a8e103aa968672e1214fc7352fec069cd36f" +version = "0.14.5" dependencies = [ "aleo-std", "anyhow", @@ -3521,9 +3449,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-block" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07d0571e7089ed6f756639d0d6878b477fe2659f79d989aa356df05a0b05b98" +version = "0.14.5" dependencies = [ "indexmap 2.0.0", "rayon", @@ -3536,9 +3462,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-coinbase" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57f7ee893fbd259be7b10fe55a4bf92e870b24a475fe84dc4cbde67811a23cd8" +version = "0.14.5" dependencies = [ "anyhow", "bincode 1.3.3", @@ -3556,9 +3480,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-query" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4bbfb2e113e59e486e25f18452982c216225e8470d717e5ac5ca6f1221b79f" +version = "0.14.5" dependencies = [ "async-trait", "reqwest", @@ -3570,9 +3492,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-store" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a0051c278212693941c959f6494a0ae8fc2be38b7c23d5d74bdedd7e3820912" +version = "0.14.5" dependencies = [ "aleo-std", "anyhow", @@ -3593,9 +3513,7 @@ dependencies = [ [[package]] name = "snarkvm-parameters" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90615b079e5cd1754b86093caa37445b4c66b2f6cdcae0902fdd37f395e55b67" +version = "0.14.5" dependencies = [ "aleo-std", "anyhow", @@ -3618,9 +3536,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f463af912a47b3fac9d12e78a9f3fe39b285d76261bb7918d72ceeec5653e91" +version = "0.14.5" dependencies = [ "aleo-std", "indexmap 2.0.0", @@ -3639,9 +3555,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-process" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdec147a8eda547537827dbd2a3dbeec530a49a9bdfd16595a7fb77878fddd2b" +version = "0.14.5" dependencies = [ "aleo-std", "colored", @@ -3661,9 +3575,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-program" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5760c27c8dc4ec7ffa4ea9a3c432b07185a702178f5ec58a2d9144e79f4586f8" +version = "0.14.5" dependencies = [ "indexmap 2.0.0", "paste", @@ -3676,9 +3588,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-snark" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aba9ac04f9ab2970f6723f7eeb28c05e528bcfa2e3792e87d943358d99de06a" +version = "0.14.5" dependencies = [ "bincode 1.3.3", "once_cell", @@ -3690,9 +3600,7 @@ dependencies = [ [[package]] name = "snarkvm-utilities" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8f137323208194925dfce7b7309204bd4a5049e9ba00dcd4c5229d51d95d880" +version = "0.14.5" dependencies = [ "aleo-std", "anyhow", @@ -3710,9 +3618,7 @@ dependencies = [ [[package]] name = "snarkvm-utilities-derives" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47a8793f220a4553a57d09bd1ba91b11543692c46dfdf5c3f264a25804cccb2a" +version = "0.14.5" dependencies = [ "proc-macro2", "quote 1.0.32", diff --git a/Cargo.toml b/Cargo.toml index 8682c6f245..6b88b3c40a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ members = [ ] [workspace.dependencies.snarkvm] -version = "=0.14.6" +path = "../snarkVMkafka" features = [ "circuit", "console", "rocks" ] [[bin]] diff --git a/cli/src/commands/leo.pkg b/cli/src/commands/leo.pkg new file mode 100644 index 0000000000..f0c07b5632 Binary files /dev/null and b/cli/src/commands/leo.pkg differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..5966f2b5c9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +version: '2.1' +services: + + zookeeper: + image: confluentinc/cp-zookeeper:6.1.1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + + # reachable on 9092 from the host and on 29092 from inside docker compose + kafka: + image: confluentinc/cp-kafka:6.1.1 + depends_on: + - zookeeper + ports: + - '9092:9092' + expose: + - '29092' + environment: + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_MIN_INSYNC_REPLICAS: '1' + + init-kafka: + image: confluentinc/cp-kafka:6.1.1 + depends_on: + - kafka + entrypoint: [ '/bin/sh', '-c' ] + command: | + " + # blocks until kafka is reachable + kafka-topics --bootstrap-server kafka:29092 --list + + echo -e 'Creating kafka topics' + kafka-topics --bootstrap-server kafka:29092 --create --if-not-exists --topic test --replication-factor 1 --partitions 1 + + echo -e 'Successfully created the following topics:' + kafka-topics --bootstrap-server kafka:29092 --list + " diff --git a/node/src/validator/mod.rs b/node/src/validator/mod.rs index 6eda7ad5b9..daf481666d 100644 --- a/node/src/validator/mod.rs +++ b/node/src/validator/mod.rs @@ -27,7 +27,7 @@ use snarkos_node_tcp::{ use snarkvm::prelude::{ block::{Block, Header}, coinbase::ProverSolution, - store::ConsensusStorage, + store::{ConsensusStorage, helpers::kafka::KafkaProducer}, Ledger, Network, }; @@ -163,17 +163,37 @@ impl> Validator { let canon_locators = crate::helpers::get_block_locators(&self.ledger)?; // Insert the canon locators into the sync pool. self.router.sync().insert_canon_locators(canon_locators).unwrap(); + let mut producer = KafkaProducer::new(); // Start the sync loop. let validator = self.clone(); self.handles.lock().push(tokio::spawn(async move { + // Clone the validator for the new task. + let validator_for_keys = validator.clone(); + // Start the continuous key-enqueuing task. + validator_for_keys.handles.lock().push(tokio::spawn(async move { + loop { + // If the Ctrl-C handler registered the signal, stop the task. + if validator_for_keys.shutdown.load(Ordering::Relaxed) { + info!("Shutting down key enqueuing"); + break; + } + + validator_for_keys.ledger.enqueue_verifying_keys(&mut producer); + // Sleep briefly to avoid overwhelming the system or the target service. + tokio::time::sleep(Duration::from_secs(1)).await; + } + })); + loop { // If the Ctrl-C handler registered the signal, stop the node. if validator.shutdown.load(Ordering::Relaxed) { info!("Shutting down block production"); break; } - + // trace!("before enqueue_verifying_keys"); + // validator.ledger.enqueue_verifying_keys(&mut producer); + // trace!("after enqueue_verifying_keys"); // Sleep briefly to avoid triggering spam detection. tokio::time::sleep(Duration::from_secs(1)).await;