gateway: Add support for multiple nodes using the same domain

Cargo.toml

@@ -86,9 +86,11 @@ serde-duration = { path = "serde-duration" }
 dstack-mr = { path = "dstack-mr" }
 dstack-verifier = { path = "verifier", default-features = false }
 size-parser = { path = "size-parser" }
+wavekv = "1.0.0"
 
 # Core dependencies
 anyhow = { version = "1.0.97", default-features = false }
+arc-swap = "1"
 or-panic = { version = "1.0", default-features = false }
 chrono = "0.4.40"
 clap = { version = "4.5.32", features = ["derive", "string"] }
@@ -109,6 +111,7 @@ sd-notify = "0.4.5"
 jemallocator = "0.5.4"
 
 # Serialization/Parsing
+flate2 = "1.1"
 borsh = { version = "1.5.7", default-features = false, features = ["derive"] }
 bon = { version = "3.4.0", default-features = false }
 base64 = "0.22.1"
@@ -122,6 +125,7 @@ scale = { version = "3.7.4", package = "parity-scale-codec", features = [
 ] }
 serde = { version = "1.0.228", features = ["derive"], default-features = false }
 serde-human-bytes = "0.1.2"
+rmp-serde = "1.3.0"
 serde_json = { version = "1.0.140", default-features = false }
 serde_ini = "0.2.0"
 toml = "0.8.20"
@@ -145,6 +149,11 @@ hyper-util = { version = "0.1.10", features = [
     "client-legacy",
     "http1",
 ] }
+hyper-rustls = { version = "0.27", default-features = false, features = [
+    "ring",
+    "http1",
+    "tls12",
+] }
 hyperlocal = "0.9.1"
 ipnet = { version = "2.11.0", features = ["serde"] }
 reqwest = { version = "0.12.14", default-features = false, features = [
@@ -233,7 +242,6 @@ yaml-rust2 = "0.10.4"
 
 luks2 = "0.5.0"
 scopeguard = "1.2.0"
-flate2 = "1.1"
 tar = "0.4"
 
 [profile.release]

REUSE.toml

@@ -191,3 +191,12 @@ SPDX-License-Identifier = "CC0-1.0"
 path = "guest-agent/fixtures/*"
 SPDX-FileCopyrightText = "NONE"
 SPDX-License-Identifier = "CC0-1.0"
+
+[[annotations]]
+path = [
+    "gateway/test-run/e2e/certs/*",
+    "gateway/test-run/e2e/configs/*",
+    "gateway/test-run/e2e/pebble-config.json",
+]
+SPDX-FileCopyrightText = "NONE"
+SPDX-License-Identifier = "CC0-1.0"

certbot/Cargo.toml

@@ -12,9 +12,12 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 bon.workspace = true
+bytes.workspace = true
 enum_dispatch.workspace = true
 fs-err.workspace = true
 hickory-resolver.workspace = true
+http.workspace = true
+http-body-util.workspace = true
 instant-acme.workspace = true
 path-absolutize.workspace = true
 rcgen.workspace = true

certbot/cli/src/main.rs

@@ -164,7 +164,7 @@ async fn main() -> Result<()> {
     {
         use tracing_subscriber::{fmt, EnvFilter};
         let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
-        fmt().with_env_filter(filter).init();
+        fmt().with_env_filter(filter).with_ansi(false).init();
     }
     rustls::crypto::ring::default_provider()
         .install_default()

certbot/src/acme_client.rs

@@ -21,6 +21,7 @@ use tracing::{debug, error, info};
 use x509_parser::prelude::{GeneralName, Pem};
 
 use super::dns01_client::{Dns01Api, Dns01Client};
+use super::http_client::ReqwestHttpClient;
 
 /// A AcmeClient instance.
 pub struct AcmeClient {
@@ -63,7 +64,9 @@ impl AcmeClient {
         dns_txt_ttl: u32,
     ) -> Result<Self> {
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
-        let account = Account::from_credentials(credentials.credentials).await?;
+        let http_client = Box::new(ReqwestHttpClient::new()?);
+        let account =
+            Account::from_credentials_and_http(credentials.credentials, http_client).await?;
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
         Ok(Self {
             account,
@@ -81,14 +84,16 @@ impl AcmeClient {
         max_dns_wait: Duration,
         dns_txt_ttl: u32,
     ) -> Result<Self> {
-        let (account, credentials) = Account::create(
+        let http_client = Box::new(ReqwestHttpClient::new()?);
+        let (account, credentials) = Account::create_with_http(
             &NewAccount {
                 contact: &[],
                 terms_of_service_agreed: true,
                 only_return_existing: false,
             },
             acme_url,
             None,
+            http_client,
         )
         .await
         .with_context(|| format!("failed to create ACME account for {acme_url}"))?;

certbot/src/acme_client/tests.rs

@@ -10,6 +10,7 @@ async fn new_acme_client() -> Result<AcmeClient> {
     let dns01_client = Dns01Client::new_cloudflare(
         std::env::var("CLOUDFLARE_ZONE_ID").expect("CLOUDFLARE_ZONE_ID not set"),
         std::env::var("CLOUDFLARE_API_TOKEN").expect("CLOUDFLARE_API_TOKEN not set"),
+        std::env::var("CLOUDFLARE_API_URL").ok(),
     );
     let credentials =
         std::env::var("LETSENCRYPT_CREDENTIAL").expect("LETSENCRYPT_CREDENTIAL not set");

certbot/src/bot.rs

@@ -28,6 +28,7 @@ pub struct CertBotConfig {
     credentials_file: PathBuf,
     auto_create_account: bool,
     cf_api_token: String,
+    cf_api_url: Option<String>,
     cert_file: PathBuf,
     key_file: PathBuf,
     cert_dir: PathBuf,
@@ -94,8 +95,12 @@ impl CertBot {
             .trim_start_matches("*.")
             .trim_end_matches('.')
             .to_string();
-        let dns01_client =
-            Dns01Client::new_cloudflare(config.cf_api_token.clone(), base_domain).await?;
+        let dns01_client = Dns01Client::new_cloudflare(
+            base_domain,
+            config.cf_api_token.clone(),
+            config.cf_api_url.clone(),
+        )
+        .await?;
         let acme_client = match fs::read_to_string(&config.credentials_file) {
             Ok(credentials) => {
                 if acme_matches(&credentials, &config.acme_url) {

certbot/src/dns01_client.rs

@@ -72,9 +72,12 @@ pub enum Dns01Client {
 }
 
 impl Dns01Client {
-    pub async fn new_cloudflare(api_token: String, base_domain: String) -> Result<Self> {
-        Ok(Self::Cloudflare(
-            CloudflareClient::new(api_token, base_domain).await?,
-        ))
+    pub async fn new_cloudflare(
+        base_domain: String,
+        api_token: String,
+        api_url: Option<String>,
+    ) -> Result<Self> {
+        let client = CloudflareClient::new(base_domain, api_token, api_url).await?;
+        Ok(Self::Cloudflare(client))
     }
 }