diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index 0ff793c8..06cf96a9 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -52,7 +52,7 @@ jobs: AS_URL=$(./scripts/accli_wrapper.sh attestation-service health --url "https://0.0.0.0:8443" --cert-path ./certs/cert.pem 2>&1 \ | grep "attestation service is healthy and reachable on:" | awk '{print $NF}') echo "Got AS URL: ${AS_URL}" - ./scripts/accli_wrapper.sh applications run function hello-snp --as-url ${AS_URL} --as-cert-path ./certs/cert.pem --in-cvm + ./scripts/accli_wrapper.sh applications run function hello-snp --backend cvm -- --as-url ${AS_URL} --as-cert-path ./certs/cert.pem - name: "Stop attestation service in the background" run: ./scripts/accli_wrapper.sh attestation-service stop diff --git a/.gitignore b/.gitignore index 2ce5fe08..f97970d1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ venv-bm build-native build-wasm -config/ansible/inventory/vms.ini +config/ansible/inventory* datasets* diff --git a/Cargo.lock b/Cargo.lock index 57d7f1af..2c77c469 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "abe4" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "ark-bls12-381", @@ -22,7 +22,7 @@ dependencies = [ [[package]] name = "accless-finra-cloudevent-handler" -version = "0.9.1" +version = "0.9.2" dependencies = [ "cloudevents-sdk", "futures-util", @@ -38,7 +38,7 @@ dependencies = [ [[package]] name = "accless-ml-inference-cloudevent-handler" -version = "0.9.1" +version = "0.9.2" dependencies = [ "cloudevents-sdk", "futures-util", @@ -54,7 +54,7 @@ dependencies = [ [[package]] name = "accless-ml-training-cloudevent-handler" -version = "0.9.1" +version = "0.9.2" dependencies = [ "cloudevents-sdk", "futures-util", @@ -70,7 +70,7 @@ dependencies = [ [[package]] name = "accless-word-count-cloudevent-handler" -version = "0.9.1" +version = "0.9.2" dependencies = [ "cloudevents-sdk", "futures-util", @@ -86,7 +86,7 @@ dependencies = [ [[package]] name = "accli" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "base64 0.22.1", @@ -448,7 +448,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "attestation-service" -version = "0.9.1" +version = "0.9.2" dependencies = [ "abe4", "accli", @@ -2617,7 +2617,7 @@ dependencies = [ [[package]] name = "jwt" -version = "0.9.1" +version = "0.9.2" dependencies = [ "base64 0.22.1", "rsa", @@ -4508,7 +4508,7 @@ dependencies = [ [[package]] name = "template-graph" -version = "0.9.1" +version = "0.9.2" dependencies = [ "abe4", "anyhow", diff --git a/accless/libs/abe4/cpp-bindings/abe4.cpp b/accless/libs/abe4/cpp-bindings/abe4.cpp index 24d80e85..2ec47de6 100644 --- a/accless/libs/abe4/cpp-bindings/abe4.cpp +++ b/accless/libs/abe4/cpp-bindings/abe4.cpp @@ -160,6 +160,14 @@ unpackFullKey(const std::vector &full_key_bytes) { std::vector packFullKey(const std::vector &authorities, const std::vector> &partial_keys) { + if (authorities.size() != partial_keys.size()) { + std::cerr << "accless(abe4): packFullKey(): size mismatch between" + << " authorities (" << authorities.size() << ") and partial" + << "keys (" << partial_keys.size() << ")" << std::endl; + throw std::runtime_error( + "accless(abe4): size mismatch packing full key"); + } + std::map> key_map; for (size_t i = 0; i < authorities.size(); ++i) { key_map[authorities[i]] = partial_keys[i]; diff --git a/accless/libs/attestation/CMakeLists.txt b/accless/libs/attestation/CMakeLists.txt index 2e831e1d..134c8154 100644 --- a/accless/libs/attestation/CMakeLists.txt +++ b/accless/libs/attestation/CMakeLists.txt @@ -19,6 +19,7 @@ set(AZ_GUEST_ATTESTATION_INCLUDE_DIRS add_library(${CMAKE_PROJECT_TARGET} attestation.cpp ec_keypair.cpp + http_client.cpp mock.cpp mock_sgx.cpp mock_snp.cpp diff --git a/accless/libs/attestation/attestation.cpp b/accless/libs/attestation/attestation.cpp index d4ebcd2a..e1fb956c 100644 --- a/accless/libs/attestation/attestation.cpp +++ b/accless/libs/attestation/attestation.cpp @@ -10,86 +10,16 @@ #include namespace accless::attestation { -// Must match the signature libcurl expects -static size_t curlWriteCallback(char *ptr, size_t size, size_t nmemb, - void *userdata) { - auto *out = static_cast(userdata); - if (!out) { - return 0; // tells libcurl this is an error - } - - const size_t total = size * nmemb; - out->append(ptr, total); - return total; -} - // Helper for GET requests static std::string http_get(const std::string &url, const std::string &certPath) { - CURL *curl = curl_easy_init(); - if (curl == nullptr) { - throw std::runtime_error("accless(att): failed to init curl"); - } - - char errbuf[CURL_ERROR_SIZE]; - errbuf[0] = 0; - - std::string response; - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, certPath.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); - - CURLcode res = curl_easy_perform(curl); - long status = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); - curl_easy_cleanup(curl); - - if (res != CURLE_OK) { - size_t len = strlen(errbuf); - fprintf(stderr, "accless(att): curl error: "); - if (len) { - fprintf(stderr, "%s%s", errbuf, - ((errbuf[len - 1] != '\n') ? "\n" : "")); - } else { - fprintf(stderr, "%s\n", curl_easy_strerror(res)); - } - throw std::runtime_error("accless(att): curl GET error"); - } - if (status != 200) { - throw std::runtime_error( - "accless(att): GET request failed with status " + - std::to_string(status)); - } - - return response; + auto &client = http::getHttpClient(certPath); + return client.get(url); } -// Get the URL of our own attestation service (**not** MAA) -std::string getAttestationServiceUrl() { - const char *val = std::getenv("ACCLESS_AS_URL"); - if (val == nullptr) { - // This url uses https://ip:port - std::cerr << "accless(att): must set ACCLESS_AS_URL" << std::endl; - throw std::runtime_error("must set ACCLESS_AS_URL"); - } - return std::string(val); -} - -std::string getAttestationServiceCertPath() { - const char *val = std::getenv("ACCLESS_AS_CERT_PATH"); - if (val == nullptr) { - std::cerr << "accless(att): must set ACCLESS_AS_CERT_PATH" << std::endl; - throw std::runtime_error("must set ACCLESS_AS_CERT_PATH"); - } - return std::string(val); -} - -std::pair getAttestationServiceState() { - std::string asUrl = getAttestationServiceUrl(); - std::string certPath = getAttestationServiceCertPath(); +std::pair +getAttestationServiceState(const std::string &asUrl, + const std::string &certPath) { std::string url = asUrl + "/state"; std::string response = http_get(url, certPath); @@ -102,48 +32,13 @@ std::pair getAttestationServiceState() { // endpoint must be one in `/verify-snp-report` or `/verify-sgx-report`. // the report here is a JSON-string -std::string getJwtFromReport(const std::string &endpoint, +std::string getJwtFromReport(const std::string &asUrl, + const std::string &certPath, + const std::string &endpoint, const std::string &reportJson) { - std::string jwt; - - CURL *curl = curl_easy_init(); - if (!curl) { - std::cerr << "accless: failed to initialize CURL" << std::endl; - throw std::runtime_error("curl error"); - } - - std::string asUrl = getAttestationServiceUrl() + endpoint; - std::string certPath = getAttestationServiceCertPath(); - curl_easy_setopt(curl, CURLOPT_URL, asUrl.c_str()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, certPath.c_str()); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, reportJson.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, - static_cast(reportJson.size())); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &jwt); - - // TODO: set error-buffer in C++ format - - struct curl_slist *headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - // Perform the request - CURLcode res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::cerr << "accless: CURL error: " << curl_easy_strerror(res) - << std::endl; - curl_easy_cleanup(curl); - curl_slist_free_all(headers); - throw std::runtime_error("curl error"); - } - - curl_easy_cleanup(curl); - curl_slist_free_all(headers); - - return jwt; + std::string url = asUrl + endpoint; + auto &client = http::getHttpClient(certPath); + return client.postJson(url, reportJson); } std::string decryptJwt(const std::vector &encrypted, diff --git a/accless/libs/attestation/attestation.h b/accless/libs/attestation/attestation.h index 0b74c9da..c8c4edd5 100644 --- a/accless/libs/attestation/attestation.h +++ b/accless/libs/attestation/attestation.h @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -42,10 +44,35 @@ std::string buildRequestBody(const std::string "eB64, const std::string &nodeId); } // namespace utils +// Helper methods around a thread-local, re-usable HTTP client. +namespace http { +class HttpClient { + public: + explicit HttpClient(const std::string &certPath); + ~HttpClient(); + + std::string get(const std::string &url); + std::string postJson(const std::string &url, const std::string &body); + + private: + CURL *curl_{nullptr}; + std::string certPath_; + std::string response_; + char errbuf_[CURL_ERROR_SIZE]; + + void prepareRequest(); + void perform(); +}; + +HttpClient &getHttpClient(const std::string &certPath); +} // namespace http + // Mock helpers used in integration tests. namespace mock { -std::string getMockSgxAttestationJwt(); -std::string getMockSnpAttestationJwt(); +std::string getMockSgxAttestationJwt(const std::string &asUrl, + const std::string &certPath); +std::string getMockSnpAttestationJwt(const std::string &asUrl, + const std::string &certPath); } // namespace mock // SNP-related methods @@ -97,16 +124,20 @@ std::vector getReport(std::array reportData); * @param nodeId The node ID. * @return A JSON string representing the JWT. */ -std::string getAttestationJwt(const std::string &gid, +std::string getAttestationJwt(const std::string &asUrl, + const std::string &certPath, + const std::string &gid, const std::string &workflowId, const std::string &nodeId); } // namespace snp // Attestation-service methods -std::string getAttestationServiceUrl(); -std::string getAttestationServiceCertPath(); -std::pair getAttestationServiceState(); -std::string getJwtFromReport(const std::string &endpoint, +std::pair +getAttestationServiceState(const std::string &asUrl, + const std::string &certPath); +std::string getJwtFromReport(const std::string &asUrl, + const std::string &certPath, + const std::string &endpoint, const std::string &reportJson); std::string decryptJwt(const std::vector &encrypted, const std::vector &aesKey); diff --git a/accless/libs/attestation/http_client.cpp b/accless/libs/attestation/http_client.cpp new file mode 100644 index 00000000..6d69d6b5 --- /dev/null +++ b/accless/libs/attestation/http_client.cpp @@ -0,0 +1,123 @@ +#include "attestation.h" + +#include +#include +#include + +namespace accless::attestation::http { +// Must match the signature libcurl expects +static size_t curlWriteCallback(char *ptr, size_t size, size_t nmemb, + void *userdata) { + auto *out = static_cast(userdata); + if (!out) { + return 0; // tells libcurl this is an error + } + + const size_t total = size * nmemb; + out->append(ptr, total); + return total; +} + +HttpClient::HttpClient(const std::string &certPath) : certPath_(certPath) { + curl_ = curl_easy_init(); + if (!curl_) { + throw std::runtime_error("accless(att): failed to init curl"); + } + + // Set options that don’t change between requests + memset(errbuf_, 0, sizeof(errbuf_)); + curl_easy_setopt(curl_, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl_, CURLOPT_CAINFO, certPath_.c_str()); + curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, curlWriteCallback); + curl_easy_setopt(curl_, CURLOPT_WRITEDATA, &response_); + curl_easy_setopt(curl_, CURLOPT_ERRORBUFFER, errbuf_); +} + +HttpClient::~HttpClient() { + if (curl_) { + curl_easy_cleanup(curl_); + } +} + +std::string HttpClient::get(const std::string &url) { + prepareRequest(); + curl_easy_setopt(curl_, CURLOPT_HTTPGET, 1L); + curl_easy_setopt(curl_, CURLOPT_URL, url.c_str()); + + perform(); + + return response_; +} + +std::string HttpClient::postJson(const std::string &url, + const std::string &body) { + prepareRequest(); + + curl_easy_setopt(curl_, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl_, CURLOPT_POST, 1L); + curl_easy_setopt(curl_, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl_, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); + + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, headers); + + perform(); + + curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, nullptr); + curl_slist_free_all(headers); + return response_; +} + +void HttpClient::prepareRequest() { + response_.clear(); + memset(errbuf_, 0, sizeof(errbuf_)); + + // Make sure we’re not leaking POST state into a GET or vice versa + curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, nullptr); + curl_easy_setopt(curl_, CURLOPT_HTTPGET, 0L); + curl_easy_setopt(curl_, CURLOPT_POST, 0L); + curl_easy_setopt(curl_, CURLOPT_POSTFIELDS, nullptr); + curl_easy_setopt(curl_, CURLOPT_POSTFIELDSIZE, 0L); + + // WRITEDATA always points to our response_ string + curl_easy_setopt(curl_, CURLOPT_WRITEDATA, &response_); +} + +void HttpClient::perform() { + CURLcode res = curl_easy_perform(curl_); + long status = 0; + curl_easy_getinfo(curl_, CURLINFO_RESPONSE_CODE, &status); + + if (res != CURLE_OK) { + const size_t len = std::strlen(errbuf_); + std::string msg = "accless(att): curl error: "; + if (len) { + msg += errbuf_; + } else { + msg += curl_easy_strerror(res); + } + throw std::runtime_error(msg); + } + + if (status != 200) { + throw std::runtime_error( + "accless(att): HTTP request failed with status " + + std::to_string(status)); + } +} + +thread_local std::unordered_map> + tlsClients; + +HttpClient &getHttpClient(const std::string &certPath) { + auto it = tlsClients.find(certPath); + if (it == tlsClients.end()) { + it = + tlsClients.emplace(certPath, std::make_unique(certPath)) + .first; + } + return *it->second; +} +} // namespace accless::attestation::http diff --git a/accless/libs/attestation/mock_sgx.cpp b/accless/libs/attestation/mock_sgx.cpp index 1d9ce471..0075d124 100644 --- a/accless/libs/attestation/mock_sgx.cpp +++ b/accless/libs/attestation/mock_sgx.cpp @@ -10,7 +10,8 @@ constexpr size_t SGX_REPORT_DATA_SIZE = 64; const std::array MOCK_QUOTE_MAGIC_SGX = {'A', 'C', 'C', 'L', 'S', 'G', 'X', '!'}; -std::string getMockSgxAttestationJwt() { +std::string getMockSgxAttestationJwt(const std::string &asUrl, + const std::string &certPath) { // Generate ephemeral EC keypair. accless::attestation::ec::EcKeyPair keyPair; @@ -28,8 +29,8 @@ std::string getMockSgxAttestationJwt() { std::string body = utils::buildRequestBody(quoteB64, runtimeB64, MOCK_GID, MOCK_WORKFLOW_ID, MOCK_NODE_ID); - std::string response = - accless::attestation::getJwtFromReport("/verify-sgx-report", body); + std::string response = accless::attestation::getJwtFromReport( + asUrl, certPath, "/verify-sgx-report", body); std::string encryptedB64 = accless::attestation::utils::extractJsonStringField(response, "encrypted_token"); diff --git a/accless/libs/attestation/mock_snp.cpp b/accless/libs/attestation/mock_snp.cpp index 13adaa3a..1c6ae822 100644 --- a/accless/libs/attestation/mock_snp.cpp +++ b/accless/libs/attestation/mock_snp.cpp @@ -11,8 +11,9 @@ using namespace accless::attestation::ec; namespace accless::attestation::mock { -std::string getMockSnpAttestationJwt() { +std::string getMockSnpAttestationJwt(const std::string &asUrl, + const std::string &certPath) { return accless::attestation::snp::getAttestationJwt( - MOCK_GID, MOCK_WORKFLOW_ID, MOCK_NODE_ID); + asUrl, certPath, MOCK_GID, MOCK_WORKFLOW_ID, MOCK_NODE_ID); } } // namespace accless::attestation::mock diff --git a/accless/libs/attestation/snp.cpp b/accless/libs/attestation/snp.cpp index 0317c68c..0a290e6b 100644 --- a/accless/libs/attestation/snp.cpp +++ b/accless/libs/attestation/snp.cpp @@ -277,7 +277,9 @@ std::string getAsEndpoint(bool isMock) { throw std::runtime_error("No known SNP device found!"); } -std::string getAttestationJwt(const std::string &gid, +std::string getAttestationJwt(const std::string &asUrl, + const std::string &certPath, + const std::string &gid, const std::string &workflowId, const std::string &nodeId) { // Generate ephemeral EC keypair. @@ -309,8 +311,8 @@ std::string getAttestationJwt(const std::string &gid, reportB64, runtimeDataB64, gid, workflowId, nodeId); // Send the request, and get the response back. - std::string response = - accless::attestation::getJwtFromReport(getAsEndpoint(isMock), body); + std::string response = accless::attestation::getJwtFromReport( + asUrl, certPath, getAsEndpoint(isMock), body); std::string encryptedB64 = accless::attestation::utils::extractJsonStringField(response, "encrypted_token"); diff --git a/accless/libs/jwt/build.rs b/accless/libs/jwt/build.rs index 6275a85c..4f6b30d8 100644 --- a/accless/libs/jwt/build.rs +++ b/accless/libs/jwt/build.rs @@ -7,18 +7,26 @@ fn main() { let out_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let dest = out_dir.join("src").join("generated_x5c_certs.rs"); - // Optional env var with path to a PEM file. - let cert_path = env::var("ACCLESS_AS_CERT_PEM").ok(); + // Optional env var with a directory path to one or more PEM files. + let cert_dir = env::var("ACCLESS_AS_CERT_DIR").ok(); - // In the furutre we could add more certificates here. let mut entries = String::new(); - if let Some(path) = cert_path + if let Some(path) = cert_dir && !path.is_empty() { - // Rebuild if that file changes + // Rebuild if that directory changes println!("cargo:rerun-if-changed={path}"); - // Add an entry to the slice using include_str! on that path - entries.push_str(&format!(" include_str!(r\"{path}\"),\n")); + // For each entry in the directory, if it is a .pem file, add it to + // the list of certificates to embed in the binary. + for dir_entry in fs::read_dir(path).unwrap() { + let file_path = dir_entry.unwrap().path(); + if file_path.is_file() { + let file_path_str = file_path.to_str().unwrap(); + if file_path_str.ends_with(".pem") && !file_path_str.ends_with("key.pem") { + entries.push_str(&format!(" include_str!(r\"{file_path_str}\"),\n")); + } + } + } } let contents = format!( @@ -30,5 +38,5 @@ fn main() { fs::write(&dest, contents).unwrap(); // Re-run build script if env var changes - println!("cargo:rerun-if-env-changed=ACCLESS_AS_CERT_PEM"); + println!("cargo:rerun-if-env-changed=ACCLESS_AS_CERT_DIR"); } diff --git a/accless/libs/jwt/cpp-bindings/CMakeLists.txt b/accless/libs/jwt/cpp-bindings/CMakeLists.txt index 58a03d94..2329b407 100644 --- a/accless/libs/jwt/cpp-bindings/CMakeLists.txt +++ b/accless/libs/jwt/cpp-bindings/CMakeLists.txt @@ -13,12 +13,21 @@ endif () add_library(jwt_rust_lib STATIC IMPORTED GLOBAL) -# This cert path can be set from: -DACCLESS_AS_CERT_PEM=/path/to/cert.pem -message(STATUS "Injecting AS certificates to JWT library from: '${ACCLESS_AS_CERT_PEM}'") +# This cert path can be set from: -DACCLESS_AS_CERT_DIR=/path/to/cert/dir +message(STATUS "Injecting AS certificates to JWT library from: '${ACCLESS_AS_CERT_DIR}'") + +# It may be that we are building the accless library as a third-party dependency +# from another project (like when we build guest modules for Knative). In that +# case, we want to be able to inject the AS certificate path as a CMake argument +# that is then passed to the Rust build as an env. var. +set_property(TARGET jwt_rust_lib APPEND PROPERTY + CARGO_ENV + ACCLESS_AS_CERT_DIR=${ACCLESS_AS_CERT_DIR} +) add_custom_command( OUTPUT ${JWT_RUST_LIBRARY} COMMAND ${CMAKE_COMMAND} -E env - ACCLESS_AS_CERT_PEM=${ACCLESS_AS_CERT_PEM} + ACCLESS_AS_CERT_DIR=${ACCLESS_AS_CERT_DIR} cargo build -p jwt ${CARGO_FLAGS} > /dev/null 2>&1 COMMENT "Building JWT staticlib with Cargo" WORKING_DIRECTORY ${ACCLESS_ROOT}/.. diff --git a/accless/src/accless.cpp b/accless/src/accless.cpp index a23eefa1..8af5037a 100644 --- a/accless/src/accless.cpp +++ b/accless/src/accless.cpp @@ -92,8 +92,8 @@ static bool validHardwareAttestation(std::string &jwtStrOut) { std::string jwtStr(jwt); #else // FIXME: get the gid/wid/nid from the message somehow - std::string jwtStr = - accless::attestation::snp::getAttestationJwt("gid", "wid", "nid"); + std::string jwtStr = accless::attestation::snp::getAttestationJwt( + "", "", "gid", "wid", "nid"); #endif #ifdef ACCLESS_UBENCH diff --git a/accli/src/main.rs b/accli/src/main.rs index b2f67d94..f4cc8aaa 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -26,9 +26,6 @@ struct Cli { // The name of the task to execute #[clap(subcommand)] task: Command, - - #[arg(short, long, global = true)] - debug: bool, } #[derive(Debug, Subcommand)] @@ -101,6 +98,9 @@ enum AttestationServiceCommand { /// Overwrite the public IP of the attestation service. #[arg(long)] overwrite_external_ip: Option, + /// Unique ID for this attestation service instance. + #[arg(long)] + id: Option, }, /// Stop a running attestation service (started with --background). Stop {}, @@ -110,7 +110,7 @@ enum AttestationServiceCommand { url: Option, /// Path to the attestation service's public certificate PEM file #[arg(long)] - cert_path: Option, + cert_dir: Option, }, } @@ -400,7 +400,7 @@ enum ApplicationsCommand { debug: bool, /// Path to the attestation service's public certificate PEM file. #[arg(long)] - as_cert_path: Option, + as_cert_dir: Option, /// Whether to build the application inside a cVM. #[arg(long, default_value_t = false)] in_cvm: bool, @@ -411,15 +411,9 @@ enum ApplicationsCommand { app_type: applications::ApplicationType, /// Name of the application to run app_name: applications::ApplicationName, - /// Whether to run the application inside a cVM. - #[arg(long, default_value_t = false)] - in_cvm: bool, - /// URL of the attestation service to contact. - #[arg(long)] - as_url: Option, - /// Path to the attestation service's public certificate PEM file. + /// Application backend. #[arg(long)] - as_cert_path: Option, + backend: Option, /// Run the application with sudo privileges. #[arg(long, default_value_t = false)] run_as_root: bool, @@ -455,17 +449,15 @@ async fn main() -> anyhow::Result<()> { ApplicationsCommand::Build { clean, debug, - as_cert_path, + as_cert_dir, in_cvm, } => { - Applications::build(*clean, *debug, as_cert_path.clone(), false, *in_cvm)?; + Applications::build(*clean, *debug, as_cert_dir.clone(), false, *in_cvm)?; } ApplicationsCommand::Run { app_type, app_name, - in_cvm, - as_url, - as_cert_path, + backend, run_as_root, extra_docker_flags, args, @@ -473,12 +465,16 @@ async fn main() -> anyhow::Result<()> { let extra_docker_flags_str: Option> = extra_docker_flags .as_ref() .map(|flags| flags.iter().map(AsRef::as_ref).collect()); + let app_backend = if let Some(backend) = backend { + backend + } else { + &applications::ApplicationBackend::Docker + }; + Applications::run( - app_type.clone(), - app_name.clone(), - *in_cvm, - as_url.clone(), - as_cert_path.clone(), + app_type, + app_name, + app_backend, *run_as_root, extra_docker_flags_str.as_deref(), args.clone(), @@ -644,42 +640,92 @@ async fn main() -> anyhow::Result<()> { Command::Azure { az_command } => match az_command { AzureCommand::Accless { az_sub_command } => match az_sub_command { AzureSubCommand::Create {} => { + let mut as_names = vec![]; + for i in 0..experiments::ACCLESS_NUM_ATTESTATION_SERVICES { + as_names.push(format!( + "{}-{i}", + experiments::ACCLESS_ATTESTATION_SERVICE_BASE_VM_NAME + )); + } + Azure::create_snp_guest(experiments::ACCLESS_VM_NAME, "Standard_DC8as_v5")?; - Azure::create_snp_guest( - experiments::ACCLESS_ATTESTATION_SERVICE_VM_NAME, - "Standard_DC2as_v5", - )?; + for as_name in &as_names { + Azure::create_snp_guest(as_name, "Standard_DC2as_v5")?; + } Azure::create_aa(experiments::ACCLESS_MAA_NAME)?; Azure::open_vm_ports(experiments::ACCLESS_VM_NAME, &[22])?; - Azure::open_vm_ports( - experiments::ACCLESS_ATTESTATION_SERVICE_VM_NAME, - &[22, 8443], - )?; + for as_name in &as_names { + Azure::open_vm_ports(as_name, &[22, 8443])?; + } } AzureSubCommand::Provision {} => { - let server_ip = - Azure::get_vm_ip(experiments::ACCLESS_ATTESTATION_SERVICE_VM_NAME)?; + let mut as_names = vec![]; + let mut as_ips = vec![]; + for i in 0..experiments::ACCLESS_NUM_ATTESTATION_SERVICES { + let as_name = format!( + "{}-{i}", + experiments::ACCLESS_ATTESTATION_SERVICE_BASE_VM_NAME + ); + as_ips.push(Azure::get_vm_ip(&as_name)?); + as_names.push(as_name); + } + let accless_code_dir = format!( "/home/{}/{}", azure::AZURE_USERNAME, experiments::ACCLESS_VM_CODE_DIR ); - let as_cert_path = - format!("{accless_code_dir}/config/attestation-service/certs/cert.pem"); - - let vars: HashMap<&str, &str> = HashMap::from([ - ("as_ip", server_ip.as_str()), - ("accless_code_dir", accless_code_dir.as_str()), - ("as_cert_path", as_cert_path.as_str()), + let as_cert_dir = + format!("{accless_code_dir}/config/attestation-service/certs"); + let inventory: HashMap<&str, Vec> = HashMap::from([ + ("accless_as", as_names.clone()), + ( + "accless_cli", + vec![experiments::ACCLESS_VM_NAME.to_string()], + ), ]); - Azure::provision_with_ansible("accless", "accless", Some(vars))?; + + let mut vars: HashMap<&str, HashMap<&str, &str>> = HashMap::new(); + vars.insert( + experiments::ACCLESS_VM_NAME, + HashMap::from([ + ("accless_code_dir", accless_code_dir.as_str()), + ("as_cert_dir", as_cert_dir.as_str()), + ]), + ); + for (as_name, as_ip) in as_names.iter().zip(as_ips.iter()) { + vars.insert( + as_name, + HashMap::from([ + ("as_ip", as_ip.as_str()), + ("accless_code_dir", accless_code_dir.as_str()), + ("as_cert_dir", as_cert_dir.as_str()), + ]), + ); + } + Azure::provision_with_ansible( + "accless", + inventory, + &Env::ansible_root().join("accless.yaml"), + Some(vars), + )?; } AzureSubCommand::Ssh {} => { + let mut as_names = vec![]; + for i in 0..experiments::ACCLESS_NUM_ATTESTATION_SERVICES { + as_names.push(format!( + "{}-{i}", + experiments::ACCLESS_ATTESTATION_SERVICE_BASE_VM_NAME + )); + } + println!("client:"); println!("{}", Azure::build_ssh_command("accless-cvm")?); - println!("attestation server:"); - println!("{}", Azure::build_ssh_command("accless-as")?); + println!("attestation servers:"); + for as_name in &as_names { + println!("- {as_name}: {}", Azure::build_ssh_command(as_name)?); + } } AzureSubCommand::Delete {} => { Azure::delete_snp_guest("accless-cvm")?; @@ -698,10 +744,18 @@ async fn main() -> anyhow::Result<()> { AzureSubCommand::Provision {} => { let service_ip = Azure::get_vm_ip(experiments::ATTESTATION_SERVICE_VM_NAME)?; - let vars: HashMap<&str, &str> = HashMap::from([("as_ip", service_ip.as_str())]); + let inventory = HashMap::from([( + "attestation_service", + vec![experiments::ATTESTATION_SERVICE_VM_NAME.to_string()], + )]); + let vars = HashMap::from([( + experiments::ATTESTATION_SERVICE_VM_NAME, + HashMap::from([("as_ip", service_ip.as_str())]), + )]); Azure::provision_with_ansible( "attestation-service", - "attestationservice", + inventory, + &Env::ansible_root().join("attestation_service.yaml"), Some(vars), )?; } @@ -732,7 +786,16 @@ async fn main() -> anyhow::Result<()> { Azure::open_vm_ports(experiments::MHSM_CLIENT_VM_NAME, &[22])?; } AzureSubCommand::Provision {} => { - Azure::provision_with_ansible("accless-mhsm", "mhsm", None)?; + let inventory = HashMap::from([( + "mhsm", + vec![experiments::MHSM_CLIENT_VM_NAME.to_string()], + )]); + Azure::provision_with_ansible( + "mhsm", + inventory, + &Env::ansible_root().join("mhsm.yaml"), + None, + )?; } AzureSubCommand::Ssh {} => { println!( @@ -753,11 +816,21 @@ async fn main() -> anyhow::Result<()> { AzureSubCommand::Provision {} => { let version = Env::get_version().unwrap(); let faasm_version = Env::get_faasm_version(); - let vars: HashMap<&str, &str> = HashMap::from([ - ("accless_version", version.as_str()), - ("faasm_version", faasm_version.as_str()), - ]); - Azure::provision_with_ansible("sgx-faasm", "sgxfaasm", Some(vars))?; + let inventory = + HashMap::from([("sgx_faasm", vec!["sgx-faasm-vm".to_string()])]); + let vars = HashMap::from([( + "sgx-faasm-vm", + HashMap::from([ + ("accless_version", version.as_str()), + ("faasm_version", faasm_version.as_str()), + ]), + )]); + Azure::provision_with_ansible( + "sgx_faasm", + inventory, + &Env::ansible_root().join("sgx_faasm.yaml"), + Some(vars), + )?; } AzureSubCommand::Ssh {} => { println!("{}", Azure::build_ssh_command("sgx-faasm-vm")?); @@ -772,9 +845,18 @@ async fn main() -> anyhow::Result<()> { } AzureSubCommand::Provision {} => { let version = Env::get_version().unwrap(); - let vars: HashMap<&str, &str> = - HashMap::from([("accless_version", version.as_str())]); - Azure::provision_with_ansible("snp-knative", "snpknative", Some(vars))?; + let inventory = + HashMap::from([("snp_knative", vec!["snp-knative-vm".to_string()])]); + let vars = HashMap::from([( + "snp-knative-vm", + HashMap::from([("accless_version", version.as_str())]), + )]); + Azure::provision_with_ansible( + "snp_knative", + inventory, + &Env::ansible_root().join("snp_knative.yaml"), + Some(vars), + )?; } AzureSubCommand::Ssh {} => { println!("{}", Azure::build_ssh_command("snp-knative-vm")?); @@ -806,13 +888,28 @@ async fn main() -> anyhow::Result<()> { ); let accless_code_dir = format!("/home/{}/git/faasm/accless", azure::AZURE_USERNAME); - - let vars: HashMap<&str, &str> = HashMap::from([ - ("kbs_ip", server_ip.as_str()), - ("trustee_code_dir", trustee_code_dir.as_str()), - ("accless_code_dir", accless_code_dir.as_str()), - ]); - Azure::provision_with_ansible("accless-trustee", "trustee", Some(vars))?; + let vm_names = vec![ + experiments::TRUSTEE_CLIENT_VM_NAME.to_string(), + experiments::TRUSTEE_SERVER_VM_NAME.to_string(), + ]; + let inventory = HashMap::from([("trustee", vm_names.clone())]); + let mut vars: HashMap<&str, HashMap<&str, &str>> = HashMap::new(); + for vm_name in &vm_names { + vars.insert( + vm_name, + HashMap::from([ + ("kbs_ip", server_ip.as_str()), + ("trustee_code_dir", trustee_code_dir.as_str()), + ("accless_code_dir", accless_code_dir.as_str()), + ]), + ); + } + Azure::provision_with_ansible( + "trustee", + inventory, + &Env::ansible_root().join("trustee.yaml"), + Some(vars), + )?; } AzureSubCommand::Ssh {} => { println!("client:"); @@ -856,6 +953,7 @@ async fn main() -> anyhow::Result<()> { rebuild, background, overwrite_external_ip, + id, } => { AttestationService::run( certs_dir.as_deref(), @@ -866,13 +964,14 @@ async fn main() -> anyhow::Result<()> { *rebuild, *background, overwrite_external_ip.clone(), + id.clone(), )?; } AttestationServiceCommand::Stop {} => { AttestationService::stop()?; } - AttestationServiceCommand::Health { url, cert_path } => { - AttestationService::health(url.clone(), cert_path.clone()).await?; + AttestationServiceCommand::Health { url, cert_dir } => { + AttestationService::health(url.clone(), cert_dir.clone()).await?; } }, } diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index 15bdcc29..a05bac72 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -1,5 +1,5 @@ use crate::tasks::{ - attestation_service, cvm, + cvm, docker::{DOCKER_ACCLESS_CODE_MOUNT_DIR, Docker}, }; use anyhow::Result; @@ -11,6 +11,12 @@ use std::{ str::FromStr, }; +#[derive(Clone, Debug, ValueEnum)] +pub enum ApplicationBackend { + Cvm, + Docker, +} + #[derive(Clone, Debug, ValueEnum)] pub enum ApplicationType { Function, @@ -49,6 +55,8 @@ pub enum ApplicationName { EscrowXput, #[value(name = "hello-snp")] HelloSnp, + #[value(name = "multi-as")] + MultiAs, } impl Display for ApplicationName { @@ -59,6 +67,7 @@ impl Display for ApplicationName { ApplicationName::BreakdownSnp => write!(f, "breakdown-snp"), ApplicationName::EscrowXput => write!(f, "escrow-xput"), ApplicationName::HelloSnp => write!(f, "hello-snp"), + ApplicationName::MultiAs => write!(f, "multi-as"), } } } @@ -73,46 +82,46 @@ impl FromStr for ApplicationName { "breakdown-snp" => Ok(ApplicationName::BreakdownSnp), "escrow-xput" => Ok(ApplicationName::EscrowXput), "hello-snp" => Ok(ApplicationName::HelloSnp), + "multi-as" => Ok(ApplicationName::MultiAs), _ => anyhow::bail!("Invalid Function: {}", s), } } } -fn host_cert_path_to_target_path( - as_cert_path: &Path, - in_cvm: bool, - in_docker: bool, +pub fn host_cert_dir_to_target_path( + as_cert_dir: &Path, + backend: &ApplicationBackend, ) -> Result { - if in_cvm & in_docker { - let reason = "cannot set in_cvm and in_docker"; - error!("as_cert_path_arg_to_real_path(): {reason}"); + if !as_cert_dir.exists() { + let reason = format!( + "as certificate directory does not exist (path={})", + as_cert_dir.display() + ); + error!("host_cert_dir_to_target_path(): {reason}"); anyhow::bail!(reason); } - if !as_cert_path.exists() { + if !as_cert_dir.is_dir() { let reason = format!( - "as certificate path does not exist (path={})", - as_cert_path.display() + "as certificate path does not point to a directory (path={})", + as_cert_dir.display() ); - error!("as_cert_path_arg_to_real_path(): {reason}"); + error!("host_cert_dir_to_target_path(): {reason}"); anyhow::bail!(reason); } - if !as_cert_path.is_file() { + if as_cert_dir.read_dir()?.next().is_none() { let reason = format!( - "as certificate path does not point to a file (path={})", - as_cert_path.display() + "passed --cert-dir variable points to an empty directory: {}", + as_cert_dir.display() ); - error!("as_cert_path_arg_to_real_path(): {reason}"); + error!("host_cert_dir_to_target_path(): {reason}"); anyhow::bail!(reason); } - if in_docker { - Ok(Docker::remap_to_docker_path(as_cert_path)?) - } else if in_cvm { - Ok(cvm::remap_to_cvm_path(as_cert_path)?) - } else { - Ok(as_cert_path.to_path_buf()) + match backend { + ApplicationBackend::Cvm => Ok(cvm::remap_to_cvm_path(as_cert_dir)?), + ApplicationBackend::Docker => Ok(Docker::remap_to_docker_path(as_cert_dir)?), } } @@ -123,7 +132,7 @@ impl Applications { pub fn build( clean: bool, debug: bool, - as_cert_path: Option, + as_cert_dir: Option, capture_output: bool, in_cvm: bool, ) -> Result> { @@ -151,12 +160,13 @@ impl Applications { if in_cvm { // Make sure the certificates are available in the cVM. let mut scp_files: Vec<(PathBuf, PathBuf)> = vec![]; - if let Some(host_cert_path) = as_cert_path { - let guest_cert_path = host_cert_path_to_target_path(&host_cert_path, true, false)?; - scp_files.push((host_cert_path, guest_cert_path.clone())); + if let Some(host_cert_dir) = as_cert_dir { + let guest_cert_dir = + host_cert_dir_to_target_path(&host_cert_dir, &ApplicationBackend::Cvm)?; + scp_files.push((host_cert_dir, guest_cert_dir.clone())); - cmd.push("--as-cert-path".to_string()); - cmd.push(guest_cert_path.display().to_string()); + cmd.push("--as-cert-dir".to_string()); + cmd.push(guest_cert_dir.display().to_string()); } cvm::run( @@ -170,10 +180,11 @@ impl Applications { )?; Ok(None) } else { - if let Some(host_cert_path) = as_cert_path { - let docker_cert_path = host_cert_path_to_target_path(&host_cert_path, false, true)?; - cmd.push("--as-cert-path".to_string()); - cmd.push(docker_cert_path.display().to_string()); + if let Some(host_cert_dir) = as_cert_dir { + let docker_cert_dir = + host_cert_dir_to_target_path(&host_cert_dir, &ApplicationBackend::Docker)?; + cmd.push("--as-cert-dir".to_string()); + cmd.push(docker_cert_dir.display().to_string()); } let workdir = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR).join("applications"); let workdir_str = workdir.to_str().ok_or_else(|| { @@ -193,98 +204,71 @@ impl Applications { #[allow(clippy::too_many_arguments)] pub fn run( - app_type: ApplicationType, - app_name: ApplicationName, - in_cvm: bool, - as_url: Option, - as_cert_path: Option, + app_type: &ApplicationType, + app_name: &ApplicationName, + app_backend: &ApplicationBackend, run_as_root: bool, extra_docker_flags: Option<&[&str]>, args: Vec, ) -> anyhow::Result> { - // If --in-cvm flag is passed, we literally re run the same `accli` command, but - // inside the cVM. - if in_cvm { - let mut cmd = vec![ - "./scripts/accli_wrapper.sh".to_string(), - "applications".to_string(), - "run".to_string(), - format!("{app_type}"), - format!("{app_name}"), - ]; + match app_backend { + // If --in-cvm flag is passed, we literally re run the same `accli` command, but + // inside the cVM. + ApplicationBackend::Cvm => { + let mut cmd = vec![ + "./scripts/accli_wrapper.sh".to_string(), + "applications".to_string(), + "run".to_string(), + format!("{app_type}"), + format!("{app_name}"), + ]; - if let Some(as_url) = as_url { - cmd.push("--as-url".to_string()); - cmd.push(as_url.to_string()); - } + if run_as_root { + cmd.push("--run-as-root".to_string()); + } - if let Some(host_cert_path) = as_cert_path { - cmd.push("--as-cert-path".to_string()); - cmd.push( - host_cert_path_to_target_path(&host_cert_path, true, false)? - .display() - .to_string(), - ); - } + if !args.is_empty() { + cmd.push("--".to_string()); + cmd.extend(args); + } - if run_as_root { - cmd.push("--run-as-root".to_string()); - } + // We don't need to SCP any files here, because we assume that the certificates + // have been copied during the build stage, and persisted in the + // disk image. + cvm::run(&cmd, None, None)?; - if !args.is_empty() { - cmd.push("--".to_string()); - cmd.extend(args); + Ok(None) } + ApplicationBackend::Docker => { + let dir_name = match app_type { + ApplicationType::Function => "functions", + ApplicationType::Test => "test", + }; + // Path matches CMake build directory: + // ./applications/build-natie/{functions,test,workflows}/{name}/{binary_name} + let binary_path = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) + .join("applications/build-native") + .join(dir_name) + .join(format!("{app_name}")) + .join(format!("{app_name}")); - // We don't need to SCP any files here, because we assume that the certificates - // have been copied during the build stage, and persisted in the - // disk image. - cvm::run(&cmd, None, None)?; - - Ok(None) - } else { - let dir_name = match app_type { - ApplicationType::Function => "functions", - ApplicationType::Test => "test", - }; - // Path matches CMake build directory: - // ./applications/build-natie/{functions,test,workflows}/{name}/{binary_name} - let binary_path = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) - .join("applications/build-native") - .join(dir_name) - .join(format!("{app_name}")) - .join(format!("{app_name}")); - - let binary_path_str = binary_path.to_str().ok_or_else(|| { - anyhow::anyhow!("Binary path is not valid UTF-8: {}", binary_path.display()) - })?; - let mut cmd = if run_as_root { - vec!["sudo".to_string(), binary_path_str.to_string()] - } else { - vec![binary_path_str.to_string()] - }; - cmd.extend(args); - - let as_env_vars: Vec = match (as_url, as_cert_path) { - (Some(as_url), Some(host_cert_path)) => { - let docker_cert_path = - host_cert_path_to_target_path(&host_cert_path, false, true)? - .display() - .to_string(); - attestation_service::get_as_env_vars(&as_url, &docker_cert_path) - } - _ => vec![], - }; + let binary_path_str = binary_path.to_str().ok_or_else(|| { + let reason = format!( + "binary path is not valid UTF-8 (path={})", + binary_path.display() + ); + error!("run(): {reason}"); + anyhow::anyhow!(reason) + })?; + let mut cmd = if run_as_root { + vec!["sudo".to_string(), binary_path_str.to_string()] + } else { + vec![binary_path_str.to_string()] + }; + cmd.extend(args); - Docker::run( - &cmd, - true, - None, - &as_env_vars, - true, - false, - extra_docker_flags, - ) + Docker::run(&cmd, true, None, &[], true, false, extra_docker_flags) + } } } } diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs index 3799249c..74f1ae5c 100644 --- a/accli/src/tasks/attestation_service.rs +++ b/accli/src/tasks/attestation_service.rs @@ -13,15 +13,15 @@ use std::{ }; const AS_URL_ENV_VAR: &str = "ACCLESS_AS_URL"; -const AS_CERT_PATH_ENV_VAR: &str = "ACCLESS_AS_CERT_PATH"; +const AS_CERT_DIR_ENV_VAR: &str = "ACCLESS_AS_CERT_DIR"; const PID_FILE_PATH: &str = "./config/attestation-service/PID"; /// Returns the required attestation-service env. vars given a URL and cert /// path. -pub fn get_as_env_vars(as_url: &str, as_cert_path: &str) -> Vec { +pub fn get_as_env_vars(as_url: &str, as_cert_dir: &str) -> Vec { vec![ format!("{AS_URL_ENV_VAR}={as_url}"), - format!("{AS_CERT_PATH_ENV_VAR}={as_cert_path}"), + format!("{AS_CERT_DIR_ENV_VAR}={as_cert_dir}"), ] } @@ -51,6 +51,7 @@ impl AttestationService { rebuild: bool, background: bool, overwrite_external_ip: Option, + id: Option, ) -> Result<()> { if rebuild { Self::build()?; @@ -80,6 +81,9 @@ impl AttestationService { if let Some(ip) = overwrite_external_ip { cmd.arg("--overwrite-external-ip").arg(ip); } + if let Some(id) = id { + cmd.arg("--id").arg(id); + } if background { info!("run(): running attestation service in background..."); @@ -127,10 +131,10 @@ impl AttestationService { Ok(()) } - pub async fn health(url: Option, cert_path: Option) -> Result<()> { + pub async fn health(url: Option, cert_dir: Option) -> Result<()> { let url = url.or_else(|| std::env::var(AS_URL_ENV_VAR).ok()); - let cert_path = cert_path.or_else(|| { - std::env::var(AS_CERT_PATH_ENV_VAR) + let cert_dir = cert_dir.or_else(|| { + std::env::var(AS_CERT_DIR_ENV_VAR) .ok() .map(std::path::PathBuf::from) }); @@ -142,13 +146,19 @@ impl AttestationService { } }; - let client = match cert_path { - Some(cert_path) => { - let cert = fs::read(cert_path)?; - let cert = reqwest::Certificate::from_pem(&cert)?; - reqwest::Client::builder() - .add_root_certificate(cert) - .build() + let client = match cert_dir { + Some(cert_dir) => { + let mut client_builder = reqwest::Client::builder(); + for entry in fs::read_dir(cert_dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() && path.extension().is_some_and(|s| s == "pem") { + let cert = fs::read(&path)?; + let cert = reqwest::Certificate::from_pem(&cert)?; + client_builder = client_builder.add_root_certificate(cert); + } + } + client_builder.build() } None => reqwest::Client::builder().build(), }?; diff --git a/accli/src/tasks/azure.rs b/accli/src/tasks/azure.rs index c2ebe477..9474315b 100644 --- a/accli/src/tasks/azure.rs +++ b/accli/src/tasks/azure.rs @@ -9,6 +9,7 @@ use shellexpand; use std::{ collections::HashMap, fs, + path::Path, process::{Command, ExitStatus}, time::Duration, }; @@ -562,33 +563,42 @@ impl Azure { // Ansible Provisioning Functions // ------------------------------------------------------------------------- - // WARNING: this method assumes that the VM names are prefixed with the - // VM deployment group name - // WARNING: this method assumes that the inventory name is the same than - // the yaml file containing the tasks pub fn provision_with_ansible( - vm_deployment: &str, - inventory_name: &str, - extra_vars: Option>, + deployment_name: &str, + inventory_contents: HashMap<&str, Vec>, + tasks_yaml_file: &Path, + extra_vars: Option>>, ) -> Result<()> { - let mut inventory_file = Env::ansible_root().join("inventory"); + let mut inventory_file = Env::ansible_root().join(format!("inventory-{deployment_name}")); fs::create_dir_all(&inventory_file)?; inventory_file.push("vms.ini"); info!( - "provision_with_ansible(): provisioning VM deployment (name={vm_deployment}, inv_file={}, extra_vars={extra_vars:?})", + "provision_with_ansible(): provisioning VM deployment (name={deployment_name}, inv_file={}, extra_vars={extra_vars:?})", inventory_file.display() ); - let mut inventory = vec![format!("[{inventory_name}]")]; - let vms: Vec = Self::list_all_resources("vm", Some(vm_deployment))?; - for vm in vms { - let name = vm["name"].as_str().unwrap(); - let ip = Self::get_vm_ip(name)?; - inventory.push(format!( - "{} ansible_host={} ansible_user={}", - name, ip, AZURE_USERNAME - )); + let mut inventory = vec![]; + for (group, vm_names) in inventory_contents { + inventory.push(format!("[{group}]")); + for vm_name in &vm_names { + let vm_ip = Self::get_vm_ip(vm_name)?; + let mut entry = format!( + "{} ansible_host={} ansible_user={}", + vm_name, vm_ip, AZURE_USERNAME + ); + if let Some(extra_vars) = &extra_vars + && let Some(vm_vars) = extra_vars.get(vm_name.as_str()) + { + let vm_vars_str = vm_vars + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(" "); + entry = format!("{entry} {vm_vars_str}"); + } + inventory.push(entry); + } } fs::write(&inventory_file, inventory.join("\n") + "\n") @@ -603,10 +613,7 @@ impl Azure { "ANSIBLE_CONFIG={} ansible-playbook -i {} {} {}", Env::ansible_root().join("ansible.cfg").to_str().unwrap(), inventory_file.to_str().unwrap(), - Env::ansible_root() - .join(format!("{inventory_name}.yaml")) - .to_str() - .unwrap(), + tasks_yaml_file.display(), match extra_vars { Some(val) => { let json = serde_json::to_string(&val).unwrap(); diff --git a/accli/src/tasks/experiments/baselines.rs b/accli/src/tasks/experiments/baselines.rs index a05120b6..84cb9cf6 100644 --- a/accli/src/tasks/experiments/baselines.rs +++ b/accli/src/tasks/experiments/baselines.rs @@ -72,8 +72,9 @@ impl SystemBaseline { pub enum EscrowBaseline { Trustee, ManagedHSM, - AcclessMaa, Accless, + AcclessMaa, + AcclessSingleAuth, } impl fmt::Display for EscrowBaseline { @@ -81,8 +82,9 @@ impl fmt::Display for EscrowBaseline { match self { EscrowBaseline::Trustee => write!(f, "trustee"), EscrowBaseline::ManagedHSM => write!(f, "managed-hsm"), - EscrowBaseline::AcclessMaa => write!(f, "accless-maa"), EscrowBaseline::Accless => write!(f, "accless"), + EscrowBaseline::AcclessMaa => write!(f, "accless-maa"), + EscrowBaseline::AcclessSingleAuth => write!(f, "accless-single-auth"), } } } @@ -94,8 +96,9 @@ impl FromStr for EscrowBaseline { match input { "trustee" => Ok(EscrowBaseline::Trustee), "managed-hsm" => Ok(EscrowBaseline::ManagedHSM), - "accless-maa" => Ok(EscrowBaseline::AcclessMaa), "accless" => Ok(EscrowBaseline::Accless), + "accless-maa" => Ok(EscrowBaseline::AcclessMaa), + "accless-single-auth" => Ok(EscrowBaseline::AcclessSingleAuth), _ => Err(()), } } @@ -103,11 +106,12 @@ impl FromStr for EscrowBaseline { impl EscrowBaseline { pub fn iter_variants() -> std::slice::Iter<'static, EscrowBaseline> { - static VARIANTS: [EscrowBaseline; 4] = [ + static VARIANTS: [EscrowBaseline; 5] = [ EscrowBaseline::Trustee, EscrowBaseline::ManagedHSM, - EscrowBaseline::AcclessMaa, EscrowBaseline::Accless, + EscrowBaseline::AcclessMaa, + EscrowBaseline::AcclessSingleAuth, ]; VARIANTS.iter() } @@ -116,8 +120,10 @@ impl EscrowBaseline { match self { EscrowBaseline::Trustee => get_color_from_label("dark-orange"), EscrowBaseline::ManagedHSM => get_color_from_label("dark-green"), - EscrowBaseline::AcclessMaa => get_color_from_label("dark-blue"), EscrowBaseline::Accless => get_color_from_label("accless"), + EscrowBaseline::AcclessMaa | EscrowBaseline::AcclessSingleAuth => { + get_color_from_label("dark-blue") + } } } } diff --git a/accli/src/tasks/experiments/mod.rs b/accli/src/tasks/experiments/mod.rs index e4c5af64..664fc849 100644 --- a/accli/src/tasks/experiments/mod.rs +++ b/accli/src/tasks/experiments/mod.rs @@ -14,7 +14,8 @@ pub mod workflows; pub const ACCLESS_MAA_NAME: &str = "accless"; pub const ACCLESS_VM_CODE_DIR: &str = "git/faasm/accless"; pub const ACCLESS_VM_NAME: &str = "accless-cvm"; -pub const ACCLESS_ATTESTATION_SERVICE_VM_NAME: &str = "accless-as"; +pub const ACCLESS_ATTESTATION_SERVICE_BASE_VM_NAME: &str = "accless-as"; +pub const ACCLESS_NUM_ATTESTATION_SERVICES: usize = 3; pub const ATTESTATION_SERVICE_VM_NAME: &str = "attestation-service"; diff --git a/accli/src/tasks/experiments/plot.rs b/accli/src/tasks/experiments/plot.rs index 21c57861..46529480 100644 --- a/accli/src/tasks/experiments/plot.rs +++ b/accli/src/tasks/experiments/plot.rs @@ -1008,6 +1008,11 @@ fn plot_escrow_xput(data_files: &Vec) { // Collect data let mut data = BTreeMap::::new(); for baseline in EscrowBaseline::iter_variants() { + // Temporarily skip plotting Accless Maa. + if baseline == &EscrowBaseline::AcclessMaa { + continue; + } + data.insert(baseline.clone(), [0.0; REQUEST_COUNTS_TRUSTEE.len()]); } @@ -1021,6 +1026,11 @@ fn plot_escrow_xput(data_files: &Vec) { let file_name_len = file_name.len(); let baseline: EscrowBaseline = file_name[0..file_name_len - 4].parse().unwrap(); + // For the moment we do not plot the AcclessMaa baseline. + if baseline == EscrowBaseline::AcclessMaa { + continue; + }; + // Open the CSV and deserialize records let mut reader = ReaderBuilder::new() .has_headers(true) @@ -1037,9 +1047,10 @@ fn plot_escrow_xput(data_files: &Vec) { count += 1; let n_req = record.num_requests; let request_counts = match baseline { - EscrowBaseline::Trustee | EscrowBaseline::Accless | EscrowBaseline::AcclessMaa => { - REQUEST_COUNTS_TRUSTEE - } + EscrowBaseline::Trustee + | EscrowBaseline::Accless + | EscrowBaseline::AcclessMaa + | EscrowBaseline::AcclessSingleAuth => REQUEST_COUNTS_TRUSTEE, EscrowBaseline::ManagedHSM => REQUEST_COUNTS_MHSM, }; let idx = request_counts @@ -1109,6 +1120,7 @@ fn plot_escrow_xput(data_files: &Vec) { .unwrap(); for (baseline, values) in data { + info!("{baseline}"); // Draw line chart .draw_series(LineSeries::new( @@ -1120,7 +1132,10 @@ fn plot_escrow_xput(data_files: &Vec) { match baseline { EscrowBaseline::Trustee | EscrowBaseline::Accless - | EscrowBaseline::AcclessMaa => REQUEST_COUNTS_TRUSTEE[x] as i32, + | EscrowBaseline::AcclessMaa + | EscrowBaseline::AcclessSingleAuth => { + REQUEST_COUNTS_TRUSTEE[x] as i32 + } EscrowBaseline::ManagedHSM => REQUEST_COUNTS_MHSM[x] as i32, }, *y, @@ -1144,7 +1159,8 @@ fn plot_escrow_xput(data_files: &Vec) { match baseline { EscrowBaseline::Trustee | EscrowBaseline::Accless - | EscrowBaseline::AcclessMaa => { + | EscrowBaseline::AcclessMaa + | EscrowBaseline::AcclessSingleAuth => { REQUEST_COUNTS_TRUSTEE[x] as i32 } EscrowBaseline::ManagedHSM => REQUEST_COUNTS_MHSM[x] as i32, @@ -1175,13 +1191,13 @@ fn plot_escrow_xput(data_files: &Vec) { match baseline { EscrowBaseline::ManagedHSM => (legend_x_start, legend_y_pos), - EscrowBaseline::AcclessMaa => (legend_x_start + 220, legend_y_pos), + EscrowBaseline::Trustee => (legend_x_start + 220, legend_y_pos), _ => panic!(), } } // NOTE: we combine the labels with the figure that is placed side-by-side - for baseline in [EscrowBaseline::ManagedHSM, EscrowBaseline::AcclessMaa] { + for baseline in [EscrowBaseline::ManagedHSM, EscrowBaseline::Trustee] { // Calculate position for each legend item let (x_pos, y_pos) = legend_label_pos_for_baseline(&baseline); @@ -1233,27 +1249,26 @@ fn plot_escrow_cost() { // Rounded monthly cost (in USD) of a Standard_DCas_v5 as of 17/04/2025 const UNIT_MONTHLY_COST_DC2: u32 = 62; - // We obtain this numbers by following the instructions to run the escrow-xput - // benchmark, and do a run where we modify the number of requests to be [0, 10] - // These numbers are run when the server is a D2 + // FIXME: grab this values from the escrow-xput run. const ACCLESS_LATENCY_D2: &[f64] = &[ 0.091594, 0.100394, 0.107495, 0.111224, 0.11829, 0.122359, 0.126534, 0.131722, 0.127414, 0.123334, ]; + let trustee_latency_single_req = 0.05; // Variables: - let trustee_latency_single_req = 0.05; let trustee_unit_cost = UNIT_MONTHLY_COST_DC2; - let accless_unit_cost = UNIT_MONTHLY_COST_DC2; + let accless_unit_cost = UNIT_MONTHLY_COST_DC2 * 3; let num_max_users = 10; let accless_latency: Vec<(u32, f64)> = (1..=ACCLESS_LATENCY_D2.len() as u32) .map(|x| (x, ACCLESS_LATENCY_D2[x as usize - 1])) .collect(); - let plot_path = Env::experiments_root() + let mut plot_path = Env::experiments_root() .join(Experiment::ESCROW_COST_NAME) - .join("plots") - .join("cost.svg"); + .join("plots"); + fs::create_dir_all(plot_path.clone()).unwrap(); + plot_path.push(format!("{}.svg", Experiment::ESCROW_COST_NAME)); let root = SVGBackend::new(&plot_path, (400, 300)).into_drawing_area(); root.fill(&WHITE).unwrap(); @@ -1402,17 +1417,17 @@ fn plot_escrow_cost() { .unwrap(); fn legend_label_pos_for_baseline(baseline: &EscrowBaseline) -> (i32, i32) { - let legend_x_start = 100; + let legend_x_start = 20; let legend_y_pos = 6; match baseline { - EscrowBaseline::Trustee => (legend_x_start, legend_y_pos), - EscrowBaseline::Accless => (legend_x_start + 120, legend_y_pos), + EscrowBaseline::Accless => (legend_x_start, legend_y_pos), + EscrowBaseline::AcclessSingleAuth => (legend_x_start + 120, legend_y_pos), _ => panic!(), } } - for baseline in &[EscrowBaseline::Trustee, EscrowBaseline::Accless] { + for baseline in &[EscrowBaseline::Accless, EscrowBaseline::AcclessSingleAuth] { // Calculate position for each legend item let (x_pos, y_pos) = legend_label_pos_for_baseline(baseline); @@ -1444,7 +1459,10 @@ fn plot_escrow_cost() { } root.present().unwrap(); - println!("invrs: generated plot at: {}", plot_path.display()); + info!( + "plot_escrow_cost(): generated plot at: {}", + plot_path.display() + ); } pub fn plot(exp: &Experiment) -> Result<()> { diff --git a/accli/src/tasks/experiments/ubench.rs b/accli/src/tasks/experiments/ubench.rs index ef62ad32..74c5272d 100644 --- a/accli/src/tasks/experiments/ubench.rs +++ b/accli/src/tasks/experiments/ubench.rs @@ -1,8 +1,12 @@ use crate::{ env::Env, tasks::{ - applications::{ApplicationName, ApplicationType, Applications}, + applications::{ + ApplicationBackend, ApplicationName, ApplicationType, Applications, + host_cert_dir_to_target_path, + }, azure::{self, Azure}, + docker::Docker, experiments::{self, Experiment, baselines::EscrowBaseline}, }, }; @@ -82,7 +86,7 @@ async fn set_resource_policy(escrow_url: &str) -> Result<()> { r#" package policy default allow = false -allow {{ +allow if {{ input["submods"]["cpu"]["ear.veraison.annotated-evidence"]["{}"] }} "#, @@ -218,7 +222,9 @@ async fn measure_requests_latency( let owned_escrow_url = escrow_url.to_string(); tokio::spawn(wrap_key_in_mhsm(owned_escrow_url)) } - EscrowBaseline::Accless | EscrowBaseline::AcclessMaa => { + EscrowBaseline::Accless + | EscrowBaseline::AcclessMaa + | EscrowBaseline::AcclessSingleAuth => { panic!("accless-based baselines must be run from different script") } }) @@ -262,7 +268,9 @@ async fn run_escrow_ubench(escrow_url: &str, run_args: &UbenchRunArgs) -> Result let request_counts = match run_args.baseline { EscrowBaseline::Trustee => REQUEST_COUNTS_TRUSTEE, EscrowBaseline::ManagedHSM => REQUEST_COUNTS_MHSM, - EscrowBaseline::Accless | EscrowBaseline::AcclessMaa => REQUEST_COUNTS_ACCLESS, + EscrowBaseline::Accless + | EscrowBaseline::AcclessMaa + | EscrowBaseline::AcclessSingleAuth => REQUEST_COUNTS_ACCLESS, }; match run_args.baseline { @@ -278,14 +286,25 @@ async fn run_escrow_ubench(escrow_url: &str, run_args: &UbenchRunArgs) -> Result } } // The Accless baselines run a function that performs SKR and CP-ABE keygen. - EscrowBaseline::Accless => { - // This path is hard-coded during the Ansible provisioning of the - // attestation-service. - let cert_path = Env::proj_root() - .join("config") - .join("attestation-service") - .join("certs") - .join("cert.pem"); + EscrowBaseline::Accless | EscrowBaseline::AcclessSingleAuth => { + // These paths are hard-coded during the Ansible provisioning of + // the attestation-service. + let mut cert_paths = vec![]; + let cert_path_base = host_cert_dir_to_target_path( + &Env::proj_root() + .join("config") + .join("attestation-service") + .join("certs"), + &ApplicationBackend::Docker, + )?; + for i in 0..(escrow_url.matches(",").count() + 1) { + cert_paths.push( + cert_path_base + .join(format!("accless-as-{i}.pem")) + .display() + .to_string(), + ); + } let num_reqs = request_counts .iter() .map(|n| n.to_string()) @@ -293,22 +312,26 @@ async fn run_escrow_ubench(escrow_url: &str, run_args: &UbenchRunArgs) -> Result .join(","); Applications::run( - ApplicationType::Function, - ApplicationName::EscrowXput, - false, - Some(format!("https://{escrow_url}:8443")), - Some(cert_path), + &ApplicationType::Function, + &ApplicationName::EscrowXput, + &ApplicationBackend::Docker, false, None, vec![ + "--as-urls".to_string(), + escrow_url.to_string(), + "--as-cert-paths".to_string(), + cert_paths.join(","), "--num-warmup-repeats".to_string(), run_args.num_warmup_repeats.to_string(), "--num-repeats".to_string(), run_args.num_repeats.to_string(), "--num-requests".to_string(), num_reqs, - "--results_file".to_string(), - results_file.display().to_string(), + "--results-file".to_string(), + Docker::remap_to_docker_path(&results_file)? + .display() + .to_string(), ], )?; } @@ -320,11 +343,9 @@ async fn run_escrow_ubench(escrow_url: &str, run_args: &UbenchRunArgs) -> Result .join(","); Applications::run( - ApplicationType::Function, - ApplicationName::EscrowXput, - false, - None, - None, + &ApplicationType::Function, + &ApplicationName::EscrowXput, + &ApplicationBackend::Docker, true, // Must run application as root. // When running the Accless MAA baseline, we need additional access to the TPM // logs, as required by the Azure library. We currently don't need these for @@ -394,18 +415,32 @@ pub async fn run(ubench: &Experiment, run_args: &UbenchRunArgs) -> Result<()> { format!("{}", run_args.baseline), ]; - let client_vm_name = match run_args.baseline { + let client_vm_name = match &run_args.baseline { EscrowBaseline::Trustee => { cmd_in_vm.push("--escrow-url".to_string()); cmd_in_vm.push(Azure::get_vm_ip(experiments::TRUSTEE_SERVER_VM_NAME)?); experiments::TRUSTEE_CLIENT_VM_NAME } - EscrowBaseline::Accless => { + baseline @ (EscrowBaseline::Accless | EscrowBaseline::AcclessSingleAuth) => { + // Decide number of attestation-services based on baseline. + let num_as = if matches!(baseline, EscrowBaseline::Accless) { + experiments::ACCLESS_NUM_ATTESTATION_SERVICES + } else { + 1 + }; + let mut as_urls = vec![]; + + for i in 0..num_as { + let as_ip = Azure::get_vm_ip(&format!( + "{}-{i}", + experiments::ACCLESS_ATTESTATION_SERVICE_BASE_VM_NAME + ))?; + as_urls.push(format!("https://{as_ip}:8443")); + } + cmd_in_vm.push("--escrow-url".to_string()); - cmd_in_vm.push(Azure::get_vm_ip( - experiments::ACCLESS_ATTESTATION_SERVICE_VM_NAME, - )?); + cmd_in_vm.push(as_urls.join(",")); experiments::ACCLESS_VM_NAME } diff --git a/applications/build.py b/applications/build.py index 1821da77..f41355ac 100644 --- a/applications/build.py +++ b/applications/build.py @@ -1,6 +1,6 @@ import argparse from faasmtools.compile_util import wasm_cmake, wasm_copy_upload -from os import environ, makedirs +from os import environ, makedirs, listdir from os.path import abspath, dirname, exists, join, realpath from shutil import rmtree from subprocess import run @@ -10,7 +10,7 @@ PROJ_ROOT = dirname(APPS_ROOT) -def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=None): +def compile(wasm=False, native=False, debug=False, clean=False, as_cert_dir=None): """ Compile the different applications supported in Accless. """ @@ -24,11 +24,14 @@ def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=Non if not exists(build_dir): makedirs(build_dir) - if as_cert_path is not None: - if not exists(as_cert_path): - print(f"ERROR: passed --cert-path variable but path does not exist") + if as_cert_dir is not None: + if not exists(as_cert_dir): + print(f"ERROR: passed --cert-dir variable but path does not exist") exit(1) - as_cert_path = abspath(as_cert_path) + # Add check for empty directory + if not listdir(as_cert_dir): # Check if directory is empty + print(f"WARNING: Passed --cert-dir variable points to an empty directory: {as_cert_dir}") + as_cert_dir = abspath(as_cert_dir) # if wasm: # wasm_cmake( @@ -47,7 +50,7 @@ def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=Non "-DCMAKE_BUILD_TYPE={}".format("Debug" if debug else "Release"), "-DCMAKE_C_COMPILER=/usr/bin/clang-17", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-17", - f"-DACCLESS_AS_CERT_PEM={as_cert_path}" if as_cert_path is not None else "", + f"-DACCLESS_AS_CERT_DIR={as_cert_dir}" if as_cert_dir is not None else "", APPS_ROOT, ] cmake_cmd = " ".join(cmake_cmd) @@ -71,7 +74,7 @@ def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=Non parser.add_argument( "--debug", action="store_true", help="Build in debug mode." ) - parser.add_argument("--as-cert-path", type=str, help="Path to certificate PEM file.") + parser.add_argument("--as-cert-dir", type=str, help="Path to certificate PEM file.") args = parser.parse_args() if args.clean: @@ -84,11 +87,11 @@ def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=Non wasm=True, debug=args.debug, clean=args.clean, - as_cert_path=args.as_cert_path, + as_cert_dir=args.as_cert_dir, ) compile( native=True, debug=args.debug, clean=args.clean, - as_cert_path=args.as_cert_path, + as_cert_dir=args.as_cert_dir, ) diff --git a/applications/functions/breakdown-snp/main.cpp b/applications/functions/breakdown-snp/main.cpp index 07a10eac..ec386bac 100644 --- a/applications/functions/breakdown-snp/main.cpp +++ b/applications/functions/breakdown-snp/main.cpp @@ -10,13 +10,31 @@ #include int main(int argc, char **argv) { + std::vector args(argv + 1, argv + argc); + if (args.size() != 4) { + throw std::runtime_error( + "Expected 2 arguments: --as-url and --as-cert-path"); + } + std::string asUrl; + std::string asCertPath; + for (size_t i = 0; i < args.size(); i += 2) { + if (args[i] == "--as-url") { + asUrl = args[i + 1]; + } else if (args[i] == "--as-cert-path") { + asCertPath = args[i + 1]; + } else { + throw std::runtime_error("Invalid argument: " + args[i]); + } + } + // ======================================================================= // CP-ABE Preparation // ======================================================================= // Get the ID and MPK we need to encrypt ciphertexts with attributes from // this attestation service instance. - auto [id, partialMpk] = accless::attestation::getAttestationServiceState(); + auto [id, partialMpk] = + accless::attestation::getAttestationServiceState(asUrl, asCertPath); std::string mpk = accless::abe4::packFullKey({id}, {partialMpk}); std::string gid = "baz"; @@ -62,8 +80,8 @@ int main(int argc, char **argv) { // Send the request to Accless' attestation service, and get the response // back. - auto response = - accless::attestation::getJwtFromReport("/verify-snp-report", body); + auto response = accless::attestation::getJwtFromReport( + asUrl, asCertPath, "/verify-snp-report", body); tb.checkpoint("send report to AS"); // Accless' authorization is equivalent to checking if we can decrypt diff --git a/applications/functions/escrow-xput/main.cpp b/applications/functions/escrow-xput/main.cpp index 7eedc657..6e69db40 100644 --- a/applications/functions/escrow-xput/main.cpp +++ b/applications/functions/escrow-xput/main.cpp @@ -12,7 +12,120 @@ #include #include -std::string sendSingleAcclessRequest(const std::vector &report, +#include "maa.h" + +#include "accless/abe4/abe4.h" +#include "accless/attestation/attestation.h" +#include "accless/attestation/ec_keypair.h" +#include "accless/base64/base64.h" +#include "accless/jwt/jwt.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +class ThreadPool { + public: + ThreadPool(size_t threads) : stop(false), in_flight_tasks(0) { + for (size_t i = 0; i < threads; ++i) { + workers.emplace_back([this] { + for (;;) { + std::function task; + + { + std::unique_lock lock(this->queue_mutex); + this->condition.wait(lock, [this] { + return this->stop || !this->tasks.empty(); + }); + if (this->stop && this->tasks.empty()) { + return; + } + task = std::move(this->tasks.front()); + this->tasks.pop(); + } + + task(); + + { + std::unique_lock lock(this->tasks_mutex); + this->in_flight_tasks--; + } + this->tasks_cv.notify_one(); + } + }); + } + } + + template void enqueue(F &&f, Args &&...args) { + auto task = std::bind(std::forward(f), std::forward(args)...); + { + std::unique_lock lock(queue_mutex); + + if (stop) { + throw std::runtime_error("enqueue on stopped ThreadPool"); + } + + tasks.emplace(task); + { + std::unique_lock lock(this->tasks_mutex); + this->in_flight_tasks++; + } + } + condition.notify_one(); + } + + void wait() { + std::unique_lock lock(this->tasks_mutex); + this->tasks_cv.wait(lock, + [this] { return this->in_flight_tasks == 0; }); + } + + ~ThreadPool() { + { + std::unique_lock lock(queue_mutex); + stop = true; + } + condition.notify_all(); + for (std::thread &worker : workers) { + worker.join(); + } + } + + private: + std::vector workers; + std::queue> tasks; + + std::mutex queue_mutex; + std::condition_variable condition; + bool stop; + + std::mutex tasks_mutex; + std::condition_variable tasks_cv; + size_t in_flight_tasks; +}; + +std::vector split(const std::string &s, char delimiter) { + std::vector tokens; + std::string token; + std::istringstream tokenStream(s); + while (std::getline(tokenStream, token, delimiter)) { + tokens.push_back(token); + } + return tokens; +} + +std::string sendSingleAcclessRequest(const std::string &asUrl, + const std::string &asCertPath, + const std::vector &report, const std::vector &reportData, const std::string &gid, const std::string &workflowId, @@ -25,7 +138,8 @@ std::string sendSingleAcclessRequest(const std::vector &report, // Send the request to Accless' attestation service, and get the response // back. return accless::attestation::getJwtFromReport( - accless::attestation::snp::getAsEndpoint(false), body); + asUrl, asCertPath, accless::attestation::snp::getAsEndpoint(false), + body); } /** @@ -51,15 +165,26 @@ std::string sendSingleAcclessRequest(const std::vector &report, * @param maxParallelism The number of parallel threads to use. * @return The time elapsed to run the number of requests. */ -std::chrono::duration runRequests(int numRequests, int maxParallelism) { +std::chrono::duration +runRequests(ThreadPool &pool, int numRequests, int maxParallelism, + const std::vector &asUrls, + const std::vector &asCertPaths) { // ======================================================================= // CP-ABE Preparation // ======================================================================= // Get the ID and MPK we need to encrypt ciphertexts with attributes from // this attestation service instance. - auto [id, partialMpk] = accless::attestation::getAttestationServiceState(); - std::string mpk = accless::abe4::packFullKey({id}, {partialMpk}); + std::vector ids; + std::vector partialMpks; + for (size_t i = 0; i < asUrls.size(); ++i) { + auto [id, partialMpk] = + accless::attestation::getAttestationServiceState(asUrls[i], + asCertPaths[i]); + ids.push_back(id); + partialMpks.push_back(partialMpk); + } + std::string mpk = accless::abe4::packFullKey(ids, partialMpks); std::string gid = "baz"; std::string wfId = "foo"; @@ -67,8 +192,16 @@ std::chrono::duration runRequests(int numRequests, int maxParallelism) { // Pick the simplest policy that only relies on the attributes `wf` and // `node` which are provided by the attestation-service after a succesful - // remote attestation. - std::string policy = id + ".wf:" + wfId + " & " + id + ".node:" + nodeId; + // remote attestation. We do a conjunction over all registered attestation + // services, in order to improve throughput by load-balancing. + std::string policy; + for (size_t i = 0; i < ids.size(); ++i) { + policy += "(" + ids[i] + ".wf:" + wfId + " & " + ids[i] + + ".node:" + nodeId + ")"; + if (i < ids.size() - 1) { + policy += " | "; + } + } // Generate a test ciphertext that only us, after a succesful attestation, // should be able to decrypt. @@ -87,9 +220,6 @@ std::chrono::duration runRequests(int numRequests, int maxParallelism) { std::cout << "escrow-xput: beginning benchmark. num reqs: " << numRequests << std::endl; - std::counting_semaphore semaphore(maxParallelism); - std::vector threads; - auto start = std::chrono::steady_clock::now(); // Generate ephemeral EC keypair. @@ -103,29 +233,19 @@ std::chrono::duration runRequests(int numRequests, int maxParallelism) { auto report = accless::attestation::snp::getReport(reportData); for (int i = 1; i < numRequests; ++i) { - // Limit how many threads we spawn in parallel by acquiring a semaphore. - semaphore.acquire(); - - threads.emplace_back( - [&semaphore, &report, &reportDataVec, &gid, &wfId, &nodeId]() { - auto response = sendSingleAcclessRequest(report, reportDataVec, - gid, wfId, nodeId); - // And releasing when the thread is done. - semaphore.release(); - }); + pool.enqueue(sendSingleAcclessRequest, + std::cref(asUrls[i % asUrls.size()]), + std::cref(asCertPaths[i % asCertPaths.size()]), + std::cref(report), std::cref(reportDataVec), + std::cref(gid), std::cref(wfId), std::cref(nodeId)); } // Send one request out of the loop, to easily process the result. - auto response = - sendSingleAcclessRequest(report, reportDataVec, gid, wfId, nodeId); - - // Wait for all requests to finish. We do this now, and not at the end - // to emulate the situation where we would have N independent clients. - for (auto &t : threads) { - if (t.joinable()) { - t.join(); - } - } + auto response = sendSingleAcclessRequest(asUrls[0], asCertPaths[0], report, + reportDataVec, gid, wfId, nodeId); + + // Wait for all in-flight requests to finish. + pool.wait(); // Accless' authorization is equivalent to checking if we can decrypt // the originaly ciphertext from the AS' response. @@ -174,7 +294,7 @@ std::chrono::duration runRequests(int numRequests, int maxParallelism) { << std::endl; throw std::runtime_error("escrow-xput: bad JWT"); } - std::string uskB64 = accless::abe4::packFullKey({id}, {partialUskB64}); + std::string uskB64 = accless::abe4::packFullKey({ids[0]}, {partialUskB64}); // Run decryption. std::optional decrypted_gt = @@ -201,18 +321,21 @@ std::chrono::duration runRequests(int numRequests, int maxParallelism) { void doBenchmark(const std::vector &numRequests, int numWarmupRepeats, int numRepeats, bool maa, const std::string &resultsFile, - const std::string &maaUrl) { + const std::string &maaUrl, + const std::vector &asUrls, + const std::vector &asCertPaths) { // Write elapsed time to CSV std::ofstream csvFile(resultsFile, std::ios::out); csvFile << "NumRequests,TimeElapsed\n"; - int maxParallelism = 100; + int maxParallelism = 10; + ThreadPool pool(maxParallelism); try { for (const auto &i : numRequests) { for (int j = 0; j < numWarmupRepeats; j++) { // Pre-warming is only necessary for regular Accless. if (!maa) { - runRequests(i, maxParallelism); + runRequests(pool, i, maxParallelism, asUrls, asCertPaths); } } @@ -224,7 +347,8 @@ void doBenchmark(const std::vector &numRequests, int numWarmupRepeats, // race conditions. elapsedTimeSecs = runMaaRequests(i, 10, maaUrl); } else { - elapsedTimeSecs = runRequests(i, maxParallelism); + elapsedTimeSecs = runRequests(pool, i, maxParallelism, + asUrls, asCertPaths); } csvFile << i << "," << elapsedTimeSecs.count() << '\n'; } @@ -260,6 +384,8 @@ int main(int argc, char **argv) { int numWarmupRepeats = 1; int numRepeats = 3; std::string resultsFile; + std::vector asUrls; + std::vector asCertPaths; for (int i = 1; i < argc; ++i) { std::string arg = argv[i]; @@ -306,6 +432,22 @@ int main(int argc, char **argv) { << std::endl; return 1; } + } else if (arg == "--as-urls") { + if (i + 1 < argc) { + asUrls = split(argv[++i], ','); + } else { + std::cerr << "--as-urls option requires one argument." + << std::endl; + return 1; + } + } else if (arg == "--as-cert-paths") { + if (i + 1 < argc) { + asCertPaths = split(argv[++i], ','); + } else { + std::cerr << "--as-cert-paths option requires one argument." + << std::endl; + return 1; + } } } @@ -315,13 +457,25 @@ int main(int argc, char **argv) { return 1; } + if (!maa && (asUrls.empty() || asCertPaths.empty())) { + std::cerr << "Usage: --as-urls and --as-cert-paths are mandatory when " + "--maa is not set" + << std::endl; + return 1; + } + if (numRequests.empty()) { std::cerr << "Missing mandatory argument --num-requests" << std::endl; return 1; } + if (resultsFile.empty()) { + std::cerr << "Missing mandatory argument --results-file" << std::endl; + return 1; + } + doBenchmark(numRequests, numWarmupRepeats, numRepeats, maa, resultsFile, - maaUrl); + maaUrl, asUrls, asCertPaths); return 0; } diff --git a/applications/functions/hello-snp/main.cpp b/applications/functions/hello-snp/main.cpp index 100681c6..f5659b38 100644 --- a/applications/functions/hello-snp/main.cpp +++ b/applications/functions/hello-snp/main.cpp @@ -3,6 +3,8 @@ #include "accless/jwt/jwt.h" #include +#include +#include /** * @brief Performs a single secret-key-release operation using Accless. @@ -12,10 +14,11 @@ * deployed in a genuine SNP cVM. Either in a para-virtualized environment on * Azure, or on bare-metal. */ -int doAcclessSkr() { +int doAcclessSkr(const std::string &asUrl, const std::string &asCertPath) { // Get the ID and MPK we need to encrypt ciphertexts with attributes from // this attestation service instance. - auto [id, partialMpk] = accless::attestation::getAttestationServiceState(); + auto [id, partialMpk] = + accless::attestation::getAttestationServiceState(asUrl, asCertPath); std::cout << "escrow-xput: got attesation service's state" << std::endl; std::string mpk = accless::abe4::packFullKey({id}, {partialMpk}); std::cout << "escrow-xput: packed partial MPK into full MPK" << std::endl; @@ -43,8 +46,8 @@ int doAcclessSkr() { std::cout << "escrow-xput: running remote attestation..." << std::endl; try { - const std::string jwt = - accless::attestation::snp::getAttestationJwt(gid, wfId, nodeId); + const std::string jwt = accless::attestation::snp::getAttestationJwt( + asUrl, asCertPath, gid, wfId, nodeId); if (jwt.empty()) { std::cerr << "escrow-xput: empty JWT returned" << std::endl; return 1; @@ -97,4 +100,24 @@ int doAcclessSkr() { return 0; } -int main(int argc, char **argv) { return doAcclessSkr(); } +int main(int argc, char **argv) { + std::vector args(argv + 1, argv + argc); + if (args.size() != 4) { + std::cerr << "Expected 2 arguments: --as-url and --as-cert-path" + << std::endl; + return 1; + } + std::string asUrl; + std::string asCertPath; + for (size_t i = 0; i < args.size(); i += 2) { + if (args[i] == "--as-url") { + asUrl = args[i + 1]; + } else if (args[i] == "--as-cert-path") { + asCertPath = args[i + 1]; + } else { + std::cerr << "Invalid argument: " + args[i] << std::endl; + return 1; + } + } + return doAcclessSkr(asUrl, asCertPath); +} diff --git a/applications/test/CMakeLists.txt b/applications/test/CMakeLists.txt index c372446e..c481434f 100644 --- a/applications/test/CMakeLists.txt +++ b/applications/test/CMakeLists.txt @@ -1,3 +1,4 @@ # Mocked client replicating the behaviour of SGX-Faasm during remote attestation. add_subdirectory(./att-client-sgx) add_subdirectory(./att-client-snp) +add_subdirectory(./multi-as) diff --git a/applications/test/att-client-sgx/main.cpp b/applications/test/att-client-sgx/main.cpp index 8b4e82eb..5c161427 100644 --- a/applications/test/att-client-sgx/main.cpp +++ b/applications/test/att-client-sgx/main.cpp @@ -4,15 +4,35 @@ #include "accless/jwt/jwt.h" #include +#include +#include using namespace accless::attestation::mock; -int main() { +int main(int argc, char *argv[]) { std::cout << "att-client-sgx: running test..." << std::endl; + std::vector args(argv + 1, argv + argc); + if (args.size() != 4) { + throw std::runtime_error( + "Expected 2 arguments: --as-url and --as-cert-path"); + } + std::string asUrl; + std::string asCertPath; + for (size_t i = 0; i < args.size(); i += 2) { + if (args[i] == "--as-url") { + asUrl = args[i + 1]; + } else if (args[i] == "--as-cert-path") { + asCertPath = args[i + 1]; + } else { + throw std::runtime_error("Invalid argument: " + args[i]); + } + } + // Get the ID and MPK we need to encrypt ciphertexts with attributes from // this attestation service instance. - auto [id, partialMpk] = accless::attestation::getAttestationServiceState(); + auto [id, partialMpk] = + accless::attestation::getAttestationServiceState(asUrl, asCertPath); std::cout << "att-client-sgx: got attesation service's state" << std::endl; std::string mpk = accless::abe4::packFullKey({id}, {partialMpk}); std::cout << "att-client-sgx: packed partial MPK into full MPK" @@ -37,7 +57,8 @@ int main() { std::cout << "att-client-sgx: running remote attestation..." << std::endl; try { const std::string jwt = - accless::attestation::mock::getMockSgxAttestationJwt(); + accless::attestation::mock::getMockSgxAttestationJwt(asUrl, + asCertPath); if (jwt.empty()) { std::cerr << "att-client-sgx: empty JWT returned" << std::endl; return 1; diff --git a/applications/test/att-client-snp/main.cpp b/applications/test/att-client-snp/main.cpp index 5e84e300..6dbf7f70 100644 --- a/applications/test/att-client-snp/main.cpp +++ b/applications/test/att-client-snp/main.cpp @@ -4,15 +4,35 @@ #include "accless/jwt/jwt.h" #include +#include +#include using namespace accless::attestation::mock; -int main() { +int main(int argc, char *argv[]) { std::cout << "att-client-snp: running test..." << std::endl; + std::vector args(argv + 1, argv + argc); + if (args.size() != 4) { + throw std::runtime_error( + "Expected 2 arguments: --as-url and --as-cert-path"); + } + std::string asUrl; + std::string asCertPath; + for (size_t i = 0; i < args.size(); i += 2) { + if (args[i] == "--as-url") { + asUrl = args[i + 1]; + } else if (args[i] == "--as-cert-path") { + asCertPath = args[i + 1]; + } else { + throw std::runtime_error("Invalid argument: " + args[i]); + } + } + // Get the ID and MPK we need to encrypt ciphertexts with attributes from // this attestation service instance. - auto [id, partialMpk] = accless::attestation::getAttestationServiceState(); + auto [id, partialMpk] = + accless::attestation::getAttestationServiceState(asUrl, asCertPath); std::cout << "att-client-snp: got attesation service's state" << std::endl; std::string mpk = accless::abe4::packFullKey({id}, {partialMpk}); std::cout << "att-client-snp: packed partial MPK into full MPK" @@ -37,7 +57,8 @@ int main() { std::cout << "att-client-snp: running remote attestation..." << std::endl; try { const std::string jwt = - accless::attestation::mock::getMockSnpAttestationJwt(); + accless::attestation::mock::getMockSnpAttestationJwt(asUrl, + asCertPath); if (jwt.empty()) { std::cerr << "att-client-snp: empty JWT returned" << std::endl; return 1; diff --git a/applications/test/multi-as/CMakeLists.txt b/applications/test/multi-as/CMakeLists.txt new file mode 100644 index 00000000..7177c926 --- /dev/null +++ b/applications/test/multi-as/CMakeLists.txt @@ -0,0 +1,4 @@ +add_executable(multi-as ${CMAKE_CURRENT_LIST_DIR}/main.cpp) + +target_include_directories(multi-as PRIVATE ${ACCLESS_HEADERS}) +target_link_libraries(multi-as PRIVATE accless::accless) diff --git a/applications/test/multi-as/main.cpp b/applications/test/multi-as/main.cpp new file mode 100644 index 00000000..c4541fda --- /dev/null +++ b/applications/test/multi-as/main.cpp @@ -0,0 +1,139 @@ +#include "accless/abe4/abe4.h" +#include "accless/attestation/attestation.h" +#include "accless/attestation/mock.h" +#include "accless/jwt/jwt.h" + +#include +#include +#include +#include + +using namespace accless::attestation::mock; + +std::vector split(const std::string &s, char delimiter) { + std::vector tokens; + std::string token; + std::istringstream tokenStream(s); + while (std::getline(tokenStream, token, delimiter)) { + tokens.push_back(token); + } + return tokens; +} + +int main(int argc, char *argv[]) { + std::cout << "multi-as: running test..." << std::endl; + + std::vector args(argv + 1, argv + argc); + if (args.size() != 4) { + throw std::runtime_error( + "Expected 2 arguments: --as-urls and --as-cert-paths"); + } + std::vector asUrls; + std::vector asCertPaths; + for (size_t i = 0; i < args.size(); i += 2) { + if (args[i] == "--as-urls") { + asUrls = split(args[i + 1], ','); + } else if (args[i] == "--as-cert-paths") { + asCertPaths = split(args[i + 1], ','); + } else { + throw std::runtime_error("Invalid argument: " + args[i]); + } + } + + if (asUrls.size() != asCertPaths.size()) { + throw std::runtime_error( + "Number of URLs and certificate paths must be the same"); + } + + std::vector ids; + std::vector partialMpks; + for (size_t i = 0; i < asUrls.size(); ++i) { + auto [id, partialMpk] = + accless::attestation::getAttestationServiceState(asUrls[i], + asCertPaths[i]); + ids.push_back(id); + partialMpks.push_back(partialMpk); + } + std::cout << "multi-as: got attesation services' state" << std::endl; + + std::string mpk = accless::abe4::packFullKey(ids, partialMpks); + std::cout << "multi-as: packed partial MPKs into full MPK" << std::endl; + + std::string policy; + for (size_t i = 0; i < ids.size(); ++i) { + policy += ids[i] + ".wf:" + MOCK_WORKFLOW_ID + " & " + ids[i] + + ".node:" + MOCK_NODE_ID; + if (i < ids.size() - 1) { + policy += " & "; + } + } + + std::cout << "multi-as: encrypting cp-abe with policy: " << policy + << std::endl; + auto [gt, ct] = accless::abe4::encrypt(mpk, policy); + if (gt.empty() || ct.empty()) { + std::cerr << "multi-as: error running cp-abe encryption" << std::endl; + return 1; + } + std::cout << "multi-as: ran CP-ABE encryption" << std::endl; + + std::cout << "multi-as: running remote attestation..." << std::endl; + try { + std::vector partialUsksB64; + for (size_t i = 0; i < asUrls.size(); ++i) { + const std::string jwt = + accless::attestation::mock::getMockSnpAttestationJwt( + asUrls[i], asCertPaths[i]); + if (jwt.empty()) { + std::cerr << "multi-as: empty JWT returned" << std::endl; + return 1; + } + + std::cout << "multi-as: received JWT from " << asUrls[i] + << std::endl; + if (!accless::jwt::verify(jwt)) { + std::cerr << "multi-as: JWT signature verification failed for " + << asUrls[i] << std::endl; + return 1; + } + std::cout << "multi-as: JWT signature verified for " << asUrls[i] + << std::endl; + + std::string partialUskB64 = + accless::jwt::getProperty(jwt, "partial_usk_b64"); + if (partialUskB64.empty()) { + std::cerr << "multi-as: JWT is missing 'partial_usk_b64' field" + << std::endl; + return 1; + } + partialUsksB64.push_back(partialUskB64); + } + + std::string uskB64 = accless::abe4::packFullKey(ids, partialUsksB64); + + std::optional decrypted_gt = + accless::abe4::decrypt(uskB64, MOCK_GID, policy, ct); + + if (!decrypted_gt.has_value()) { + std::cerr << "multi-as: CP-ABE decryption failed" << std::endl; + return 1; + } else if (decrypted_gt.value() != gt) { + std::cerr << "multi-as: CP-ABE decrypted ciphertexts do not" + << " match!" << std::endl; + std::cerr << "multi-as: Original GT: " << gt << std::endl; + std::cerr << "multi-as: Decrypted GT: " << decrypted_gt.value() + << std::endl; + return 1; + } + + std::cout << "multi-as: CP-ABE decryption succesful!" << std::endl; + + return 0; + } catch (const std::exception &ex) { + std::cerr << "multi-as: error: " << ex.what() << std::endl; + } catch (...) { + std::cerr << "multi-as: unexpected error" << std::endl; + } + + return 1; +} diff --git a/attestation-service/README.md b/attestation-service/README.md index 70887eb6..0c270bd4 100644 --- a/attestation-service/README.md +++ b/attestation-service/README.md @@ -44,7 +44,7 @@ need to be hard-coded in the TEEs. In particular, we need to pass the path to the certificate's PEM file to the application build process: ```bash -accli applications build --clean --cert-path /path/to/cert.pem +accli applications build --clean --cert-dir /path/to/certs_dir ``` when starting the attestation service, you may force re-generation of the diff --git a/attestation-service/src/azure_cvm.rs b/attestation-service/src/azure_cvm.rs index 81a07ab9..e2f698ea 100644 --- a/attestation-service/src/azure_cvm.rs +++ b/attestation-service/src/azure_cvm.rs @@ -350,7 +350,9 @@ pub async fn verify_snp_vtpm_report( &Tee::AzureCvm, &payload.node_data, &raw_pubkey_bytes, - ) { + ) + .await + { Ok(response) => (StatusCode::OK, Json(response)), Err(e) => { error!("error encrypting JWT (error={e:?})"); diff --git a/attestation-service/src/ecdhe.rs b/attestation-service/src/ecdhe.rs index 0dfa51e2..cf1b7dc4 100644 --- a/attestation-service/src/ecdhe.rs +++ b/attestation-service/src/ecdhe.rs @@ -124,7 +124,7 @@ pub fn generate_ecdhe_keys_and_derive_secret( Ok((my_pubkey.as_ref().to_vec(), shared_secret)) } -pub fn do_ecdhe_ke( +pub async fn do_ecdhe_ke( state: &AttestationServiceState, tee: &Tee, node_data: &NodeData, @@ -152,6 +152,7 @@ pub fn do_ecdhe_ke( &node_data.workflow_id, &node_data.node_id, ) + .await .context("do_ecdhe_ke(): error generating JWT claims")?; let header = jsonwebtoken::Header { alg: jsonwebtoken::Algorithm::RS256, diff --git a/attestation-service/src/jwt.rs b/attestation-service/src/jwt.rs index 447c4573..c65f23ce 100644 --- a/attestation-service/src/jwt.rs +++ b/attestation-service/src/jwt.rs @@ -61,8 +61,57 @@ pub struct JwtClaims { } impl JwtClaims { - /// # Description + /// Retrieve the partial USK from the cache, create if missing. /// + /// We cache partial user secret keys for scale-out requests coming from + /// the same user, for the same function, in the same workflow. + async fn get_partial_usk_bytes( + state: &AttestationServiceState, + gid: &str, + workflow_id: &str, + node_id: &str, + ) -> Result> { + let cache_key = ( + gid.to_string(), + workflow_id.to_string(), + node_id.to_string(), + ); + + // Fast path: read from the cache. + let partial_usk_bytes = { + let cache = state.partial_usk_cache.read().await; + cache.get(&cache_key).cloned() + }; + if let Some(partial_usk_bytes) = partial_usk_bytes { + return Ok(partial_usk_bytes); + } + + // Slow path: generate partial USK and populate cache. + let rng = rand::thread_rng(); + let user_attributes: Vec = vec![ + abe4::policy::UserAttribute::new(&state.id, ATTRIBUTE_WORKFLOW_LABEL, workflow_id), + abe4::policy::UserAttribute::new(&state.id, ATTRIBUTE_NODE_LABEL, node_id), + ]; + debug!( + "get_partial_usk_bytes(): generating partial USK (gid={gid}, wfid={workflow_id}, node_id={node_id})" + ); + + let user_attribute_refs: Vec<&UserAttribute> = user_attributes.iter().collect(); + let iota = abe4::scheme::iota::Iota::new(&user_attributes); + let partial_usk: PartialUSK = + abe4::scheme::keygen_partial(rng, gid, &state.partial_msk, &user_attribute_refs, &iota); + let mut partial_usk_bytes: Vec = Vec::new(); + partial_usk.serialize_compressed(&mut partial_usk_bytes)?; + + // Cache key bytes for future use. + { + let mut cache = state.partial_usk_cache.write().await; + cache.insert(cache_key, partial_usk_bytes.clone()); + } + + Ok(partial_usk_bytes) + } + /// Generates a new JWT based on the attestation service state, and the /// specific request metadata. /// @@ -75,29 +124,15 @@ impl JwtClaims { /// executing. /// - `node_id`: unique identifier of the node in the workflow graph we are /// executing. - pub fn new( + pub async fn new( state: &AttestationServiceState, tee: &Tee, gid: &str, workflow_id: &str, node_id: &str, ) -> Result { - let rng = rand::thread_rng(); - let user_attributes: Vec = vec![ - abe4::policy::UserAttribute::new(&state.id, ATTRIBUTE_WORKFLOW_LABEL, workflow_id), - abe4::policy::UserAttribute::new(&state.id, ATTRIBUTE_NODE_LABEL, node_id), - ]; - debug!("Generating partial USK for gid: {}", gid); - debug!("- Workflow ID: {}", workflow_id); - debug!("- Node ID: {}", node_id); - debug!("- User Attributes: {:?}", user_attributes); - - let user_attribute_refs: Vec<&UserAttribute> = user_attributes.iter().collect(); - let iota = abe4::scheme::iota::Iota::new(&user_attributes); - let partial_usk: PartialUSK = - abe4::scheme::keygen_partial(rng, gid, &state.partial_msk, &user_attribute_refs, &iota); - let mut partial_usk_bytes: Vec = Vec::new(); - partial_usk.serialize_compressed(&mut partial_usk_bytes)?; + let partial_usk_bytes = + Self::get_partial_usk_bytes(state, gid, workflow_id, node_id).await?; Ok(Self { sub: "attested-client".to_string(), diff --git a/attestation-service/src/main.rs b/attestation-service/src/main.rs index ffc4e3cf..10968c67 100644 --- a/attestation-service/src/main.rs +++ b/attestation-service/src/main.rs @@ -55,6 +55,9 @@ struct Cli { /// Overwrite the public IP of the attestation service. #[arg(long)] overwrite_external_ip: Option, + /// Overwrite the attestation service ID. + #[arg(long)] + id: Option, } async fn health(Extension(state): Extension>) -> impl IntoResponse { @@ -125,6 +128,7 @@ async fn main() -> Result<()> { cli.sgx_pccs_url.clone(), cli.mock, external_url.clone(), + cli.id.clone(), )?); // Start HTTPS server. diff --git a/attestation-service/src/sgx.rs b/attestation-service/src/sgx.rs index 8391f7ff..870aa4d6 100644 --- a/attestation-service/src/sgx.rs +++ b/attestation-service/src/sgx.rs @@ -282,7 +282,7 @@ pub async fn verify_sgx_report( // Now that we have verified the attestation report, run the server-side part of // the attribute minting protocol which involves running ECDHE and running // CP-ABE keygen. - match ecdhe::do_ecdhe_ke(&state, &Tee::Sgx, &payload.node_data, &raw_pubkey_bytes) { + match ecdhe::do_ecdhe_ke(&state, &Tee::Sgx, &payload.node_data, &raw_pubkey_bytes).await { Ok(response) => (StatusCode::OK, Json(response)), Err(e) => { error!("error encrypting JWT (error={e:?})"); diff --git a/attestation-service/src/snp.rs b/attestation-service/src/snp.rs index 473e0731..e020ed08 100644 --- a/attestation-service/src/snp.rs +++ b/attestation-service/src/snp.rs @@ -205,7 +205,7 @@ pub async fn verify_snp_report( // Now that we have verified the attestation report, run the server-side part of // the attribute minting protocol which involves running ECDHE and running // CP-ABE keygen. - match ecdhe::do_ecdhe_ke(&state, &Tee::Snp, &payload.node_data, &raw_pubkey_bytes) { + match ecdhe::do_ecdhe_ke(&state, &Tee::Snp, &payload.node_data, &raw_pubkey_bytes).await { Ok(response) => (StatusCode::OK, Json(response)), Err(e) => { error!("error encrypting JWT (error={e:?})"); diff --git a/attestation-service/src/state.rs b/attestation-service/src/state.rs index 14516faa..4a0e8310 100644 --- a/attestation-service/src/state.rs +++ b/attestation-service/src/state.rs @@ -37,6 +37,10 @@ pub struct AttestationServiceState { /// Master Pulic Key for the attestation service as one of the authorities /// of the decentralized CP-ABE scheme. pub partial_mpk: PartialMPK, + /// Cache of generated partial User Secret Key per GID, workflow Id, and + /// node Id. + // TODO: periodically clean the cache. + pub partial_usk_cache: RwLock>>, // Fields related to verifying attestation reports from TEEs. @@ -74,13 +78,15 @@ impl AttestationServiceState { sgx_pccs_url: Option, mock_attestation: bool, external_url: String, + id: Option, ) -> Result { let certs_dir = certs_dir.unwrap_or_else(get_default_certs_dir); // Initialize CP-ABE authority. let mut rng = rand::thread_rng(); + let id = id.unwrap_or(ATTESTATION_SERVICE_ID.to_string()); let (partial_msk, partial_mpk): (PartialMSK, PartialMPK) = - abe4::scheme::setup_partial(&mut rng, ATTESTATION_SERVICE_ID); + abe4::scheme::setup_partial(&mut rng, &id); // Fetch AMD signing keys. @@ -88,9 +94,10 @@ impl AttestationServiceState { external_url, mock_attestation, jwt_encoding_key: jwt::generate_encoding_key(&certs_dir)?, - id: ATTESTATION_SERVICE_ID.to_string(), + id, partial_msk, partial_mpk, + partial_usk_cache: RwLock::new(HashMap::new()), #[cfg(feature = "sgx")] sgx_pccs_url, #[cfg(feature = "sgx")] diff --git a/attestation-service/tests/as_api_tests.rs b/attestation-service/tests/as_api_tests.rs index 30d3b4d9..17e9632f 100644 --- a/attestation-service/tests/as_api_tests.rs +++ b/attestation-service/tests/as_api_tests.rs @@ -1,10 +1,14 @@ -use accli::tasks::applications::{ApplicationName, ApplicationType, Applications}; +use accli::tasks::applications::{ + ApplicationBackend, ApplicationName, ApplicationType, Applications, + host_cert_dir_to_target_path, +}; use anyhow::Result; use log::{error, info}; use reqwest::Client; use serde_json::Value; use serial_test::serial; use std::{ + fs, path::{Path, PathBuf}, process::Stdio, time::{Duration, Instant}, @@ -125,9 +129,6 @@ async fn test_att_clients() -> Result<()> { let child = spawn_as(certs_dir.to_str().unwrap(), true, true)?; let _child_guard = ChildGuard(child); - // Give the service time to start. - tokio::time::sleep(Duration::from_secs(2)).await; - let cert_path = get_public_certificate_path(&certs_dir); // Wait until cert path to be ready. @@ -155,7 +156,13 @@ async fn test_att_clients() -> Result<()> { // container. We also _must_ set the `clean` flag to true, to force // recompilation. info!("re-building mock clients with new certificates, this will take a while..."); - Applications::build(true, false, Some(cert_path.clone()), true, false)?; + Applications::build( + true, + false, + Some(certs_dir.clone().to_path_buf()), + true, + false, + )?; // Health-check the attestation service. let client = reqwest::Client::builder() @@ -168,33 +175,49 @@ async fn test_att_clients() -> Result<()> { info!("running mock sgx client..."); Applications::run( - ApplicationType::Test, - ApplicationName::AttClientSgx, - false, - Some(as_url.clone()), - Some(cert_path.clone()), + &ApplicationType::Test, + &ApplicationName::AttClientSgx, + &ApplicationBackend::Docker, false, None, - vec![], + vec![ + "--as-url".to_string(), + as_url.clone(), + "--as-cert-path".to_string(), + get_public_certificate_path(&host_cert_dir_to_target_path( + &certs_dir, + &ApplicationBackend::Docker, + )?) + .display() + .to_string(), + ], )?; info!("running mock snp client..."); Applications::run( - ApplicationType::Test, - ApplicationName::AttClientSnp, - false, - Some(as_url), - Some(cert_path), + &ApplicationType::Test, + &ApplicationName::AttClientSnp, + &ApplicationBackend::Docker, false, None, - vec![], + vec![ + "--as-url".to_string(), + as_url.clone(), + "--as-cert-path".to_string(), + get_public_certificate_path(&host_cert_dir_to_target_path( + &certs_dir, + &ApplicationBackend::Docker, + )?) + .display() + .to_string(), + ], )?; - match std::fs::remove_dir_all(&certs_dir) { + match fs::remove_dir_all(&certs_dir) { Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => { - error!("error removing certs dir (error={e:?}, dir={certs_dir:?})"); + error!("test_att_clients(): error removing certs dir (error={e:?}, dir={certs_dir:?})"); } } @@ -225,3 +248,116 @@ async fn test_get_state() -> Result<()> { Ok(()) } + +#[tokio::test] +#[serial] +async fn test_multi_as() -> Result<()> { + attestation_service::init_logging(); + + let certs_dir_1 = tempdir()?; + let certs_dir_2 = tempdir()?; + + let mut cmd_1 = Command::new(env!("CARGO_BIN_EXE_attestation-service")); + cmd_1.arg("--certs-dir").arg(certs_dir_1.path()); + cmd_1.arg("--port").arg("8443"); + cmd_1.arg("--id").arg("as1"); + cmd_1.arg("--force-clean-certs"); + cmd_1.arg("--mock"); + let _child1 = ChildGuard(cmd_1.spawn()?); + + let mut cmd_2 = Command::new(env!("CARGO_BIN_EXE_attestation-service")); + cmd_2.arg("--certs-dir").arg(certs_dir_2.path()); + cmd_2.arg("--port").arg("8444"); + cmd_2.arg("--id").arg("as2"); + cmd_2.arg("--force-clean-certs"); + cmd_2.arg("--mock"); + let _child2 = ChildGuard(cmd_2.spawn()?); + + let cert_path_1 = get_public_certificate_path(certs_dir_1.path()); + let cert_path_2 = get_public_certificate_path(certs_dir_2.path()); + + for cert_path in [cert_path_1.clone(), cert_path_2.clone()] { + let deadline = Instant::now() + Duration::from_secs(15); + let poll_interval = Duration::from_millis(100); + loop { + if cert_path.exists() { + break; + } + if Instant::now() >= deadline { + let reason = format!( + "timed-out waiting for certs to become available (path={})", + cert_path.display() + ); + error!("test_multi_as(): {reason}"); + anyhow::bail!(reason); + } + + sleep(poll_interval).await; + } + } + + let merged_certs_dir = Path::new(env!("ACCLESS_ROOT_DIR")) + .join("config") + .join("attestation-service") + .join("test-certs"); + fs::create_dir_all(&merged_certs_dir)?; + fs::copy(&cert_path_1, merged_certs_dir.join("cert1.pem"))?; + fs::copy(&cert_path_2, merged_certs_dir.join("cert2.pem"))?; + + info!( + "test_multi_as(): re-building multi-as client with new certificates, this will take a while..." + ); + let merged_certs_dir_docker = + host_cert_dir_to_target_path(&merged_certs_dir, &ApplicationBackend::Docker)?; + Applications::build( + true, + false, + Some(merged_certs_dir.to_path_buf()), + true, + false, + )?; + + let client1 = reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .build()?; + health_check(&client1).await?; + let client2 = reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .build()?; + health_check(&client2).await?; + + // Convert full paths to paths relative to the docker mount point. + let cert_path_1_docker = merged_certs_dir_docker.join("cert1.pem"); + let cert_path_2_docker = merged_certs_dir_docker.join("cert2.pem"); + + info!("running mock multi-as client..."); + Applications::run( + &ApplicationType::Test, + &ApplicationName::MultiAs, + &ApplicationBackend::Docker, + false, + None, + vec![ + "--as-urls".to_string(), + "https://localhost:8443,https://localhost:8444".to_string(), + "--as-cert-paths".to_string(), + format!( + "{},{}", + cert_path_1_docker.display(), + cert_path_2_docker.display() + ), + ], + )?; + + match fs::remove_dir_all(&merged_certs_dir) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + error!( + "test_multi_as(): error removing certs dir (error={e:?}, dir={merged_certs_dir:?})" + ); + } + } + + Ok(()) +} diff --git a/config/ansible/accless.yaml b/config/ansible/accless.yaml index 2bc8434c..93926a1f 100644 --- a/config/ansible/accless.yaml +++ b/config/ansible/accless.yaml @@ -1,10 +1,114 @@ --- -- hosts: accless +- name: "Set-Up Common Environment" + hosts: accless_as,accless_cli gather_facts: yes tasks: - - include_tasks: tasks/accless/apt.yaml - - include_tasks: tasks/util/docker.yaml - - include_tasks: tasks/util/rust.yaml - - include_tasks: tasks/accless/code.yaml - - include_tasks: tasks/accless.yaml + - include_tasks: tasks/util/docker.yaml + - include_tasks: tasks/util/rust.yaml + - name: "Create code dir" + file: + path: "/home/{{ ansible_user }}/git" + state: directory + - name: "Clone Accless repos" + git: + repo: "https://www.github.com/faasm/tless.git" + dest: "{{ accless_code_dir }}" + update: yes + recursive: yes + clone: yes + force: yes + accept_hostkey: yes + - name: "Install Accless' APT Dependencies" + shell: ./scripts/apt.sh + args: + chdir: "{{ accless_code_dir }}" + executable: /bin/bash + +# ============================================================================= +# Server-side configuration +# ============================================================================= + +- name: "Configure Attestation Services" + hosts: accless_as + tasks: + - name: "Build Attestation Service" + shell: ./scripts/accli_wrapper.sh attestation-service build + args: + chdir: "{{ accless_code_dir }}" + executable: /bin/bash + environment: + PATH: "/home/{{ ansible_user }}/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + + - name: "Install attestation-service systemd unit" + become: true + copy: + dest: /etc/systemd/system/attestation-service.service + content: | + [Unit] + Description=Accless Attestation Service + After=network.target + + [Service] + WorkingDirectory={{ accless_code_dir }} + ExecStart=/bin/bash -lc './scripts/accli_wrapper.sh attestation-service run --overwrite-external-ip {{ as_ip }} --id {{ inventory_hostname | b64encode }}' + Restart=on-failure + User={{ ansible_user }} + + [Install] + WantedBy=multi-user.target + + - name: "Reload systemd" + become: true + ansible.builtin.systemd: + daemon_reload: yes + + - name: "Enable & start attestation-service" + become: true + ansible.builtin.systemd: + name: attestation-service + enabled: yes + state: started + + - name: "Wait for attestation-service certificate to exist" + ansible.builtin.wait_for: + path: "{{ as_cert_dir }}/cert.pem" + state: present + timeout: 120 + + - name: "Read attestation-service certificate (slurp)" + ansible.builtin.slurp: + src: "{{ as_cert_dir }}/cert.pem" + register: as_cert_pem + +# ============================================================================= +# Client-side configuration +# ============================================================================= + +- name: "Configure Attestation Services" + hosts: accless_cli + tasks: + # First, copy the cert from accless-as's hostvars to the client filesystem. + - name: "Ensure cert directory exists on client" + ansible.builtin.file: + path: "{{ as_cert_dir }}" + state: directory + mode: "0755" + + - name: "Write attestation-service certificates on client (one per AS host)" + ansible.builtin.copy: + dest: "{{ as_cert_dir }}/{{ item }}.pem" + mode: "0600" + content: "{{ hostvars[item].as_cert_pem.content | b64decode }}" + loop: "{{ groups['accless_as'] }}" + + # Build all applications with the new certificate. Under the hood this also + # pulls the work-on container image and builds `accli`, so it may take a while. + - name: "Build applications" + when: inventory_hostname == "accless-cvm" + shell: ./scripts/accli_wrapper.sh applications build --clean --as-cert-dir {{ as_cert_dir }} + args: + chdir: "{{ accless_code_dir }}" + executable: /bin/bash + environment: + PATH: "/home/{{ ansible_user }}/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/config/ansible/ansible.cfg b/config/ansible/ansible.cfg index 2c672d95..34339d08 100644 --- a/config/ansible/ansible.cfg +++ b/config/ansible/ansible.cfg @@ -1,3 +1,4 @@ [defaults] +forks = 10 host_key_checking = False interpreter_python = auto_silent diff --git a/config/ansible/attestationservice.yaml b/config/ansible/attestation_service.yaml similarity index 100% rename from config/ansible/attestationservice.yaml rename to config/ansible/attestation_service.yaml diff --git a/config/ansible/sgxfaasm.yaml b/config/ansible/sgx_faasm.yaml similarity index 100% rename from config/ansible/sgxfaasm.yaml rename to config/ansible/sgx_faasm.yaml diff --git a/config/ansible/snpknative.yaml b/config/ansible/snp_knative.yaml similarity index 100% rename from config/ansible/snpknative.yaml rename to config/ansible/snp_knative.yaml diff --git a/config/ansible/tasks/accless.yaml b/config/ansible/tasks/accless.yaml index db2ac834..fb240f58 100644 --- a/config/ansible/tasks/accless.yaml +++ b/config/ansible/tasks/accless.yaml @@ -49,14 +49,14 @@ - name: "Wait for attestation-service certificate to exist" when: inventory_hostname == "accless-as" ansible.builtin.wait_for: - path: "{{ as_cert_path }}" + path: "{{ as_cert_dir }}/cert.pem" state: present timeout: 120 - name: "Read attestation-service certificate (slurp)" when: inventory_hostname == "accless-as" ansible.builtin.slurp: - src: "{{ as_cert_path }}" + src: "{{ as_cert_dir }}/cert.pem" register: as_cert_pem # ============================================================================= @@ -67,13 +67,13 @@ - name: "Ensure cert directory exists on client" when: inventory_hostname == "accless-cvm" ansible.builtin.file: - path: "{{ as_cert_path | dirname }}" + path: "{{ as_cert_dir }}" state: directory mode: "0755" - name: "Write attestation-service certificate on client" when: inventory_hostname == "accless-cvm" ansible.builtin.copy: - dest: "{{ as_cert_path }}" + dest: "{{ as_cert_dir }}/cert.pem" mode: "0600" content: "{{ hostvars['accless-as'].as_cert_pem.content | b64decode }}" @@ -82,7 +82,7 @@ - name: "Build applications" when: inventory_hostname == "accless-cvm" # FIXME: remove branch name - shell: git checkout enhancement-escrow-xput && ./scripts/accli_wrapper.sh applications build --clean --as-cert-path {{ as_cert_path }} + shell: git checkout enhancement-escrow-xput && ./scripts/accli_wrapper.sh applications build --clean --as-cert-dir {{ as_cert_dir }} args: chdir: "{{ accless_code_dir }}" executable: /bin/bash diff --git a/config/ansible/tasks/accless/apt.yaml b/config/ansible/tasks/accless/apt.yaml deleted file mode 100644 index 81cc3727..00000000 --- a/config/ansible/tasks/accless/apt.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: "Install APT depdencencies" - become: yes - apt: - name: - - build-essential - - libfontconfig1-dev - - libtss2-dev - - libtss2-sys1t64 - - libssl-dev - - pkg-config - - python3-venv - update_cache: yes diff --git a/config/ansible/tasks/accless/az_guest_attestation_prereq.yaml b/config/ansible/tasks/accless/az_guest_attestation_prereq.yaml deleted file mode 100644 index eb872732..00000000 --- a/config/ansible/tasks/accless/az_guest_attestation_prereq.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- name: "Install the pre-reqs" - become: yes - shell: bash pre-requisites.sh - args: - chdir: "/home/{{ ansible_user }}/git/faasm/azure-cvm-guest-attestation" - executable: /bin/bash diff --git a/config/ansible/tasks/accless/code.yaml b/config/ansible/tasks/accless/code.yaml deleted file mode 100644 index e81a38f5..00000000 --- a/config/ansible/tasks/accless/code.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- - -- name: "Create code dir" - file: - path: "/home/{{ ansible_user }}/git" - state: directory - -- name: "Clone Accless repos" - git: - repo: "https://www.github.com/faasm/tless.git" - dest: "{{ accless_code_dir }}" - update: yes - recursive: yes - clone: yes - force: yes - accept_hostkey: yes diff --git a/config/ansible/tasks/accless/rabe.yaml b/config/ansible/tasks/accless/rabe.yaml deleted file mode 100644 index f8eba021..00000000 --- a/config/ansible/tasks/accless/rabe.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: "Build rabe (CP-ABE) library and C++ bindings" - shell: ./bin/inv_wrapper.sh rabe --native - args: - chdir: "/home/{{ ansible_user }}/git/faasm/examples" - executable: /bin/bash - environment: - PATH: "/home/{{ ansible_user }}/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/config/ansible/tasks/util/docker.yaml b/config/ansible/tasks/util/docker.yaml index 45317966..a630ce91 100644 --- a/config/ansible/tasks/util/docker.yaml +++ b/config/ansible/tasks/util/docker.yaml @@ -7,7 +7,7 @@ - name: "Add Docker APT repository" become: yes apt_repository: - repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable" + repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_facts['distribution'] | lower }} {{ ansible_facts['distribution_release'] }} stable" - name: "Install Docker" become: yes diff --git a/template-graph/src/lib.rs b/template-graph/src/lib.rs index e2d2116f..e8320923 100644 --- a/template-graph/src/lib.rs +++ b/template-graph/src/lib.rs @@ -175,10 +175,9 @@ impl TemplateGraph { let graph: TemplateGraph = serde_yaml::from_str(yaml)?; if graph.authorities.attestation_services.is_empty() { - error!("Template graph must contain at least one attestation service"); - return Err(serde_yaml::Error::custom( - "`authorities.attestation-services` must not be empty.", - )); + let reason = "`authorities.attestation-services` must not be empty"; + error!("from_yaml(): {reason}"); + return Err(serde_yaml::Error::custom(reason)); } let mut aps_ids: std::collections::HashSet = graph @@ -194,11 +193,13 @@ impl TemplateGraph { for i in 0..policy.len() { let (attr, _) = policy.get(i); if !aps_ids.contains(attr.authority()) { - return Err(serde_yaml::Error::custom(format!( - "Authority '{}' in node policy for '{}' not found in attribute providing services.", + let reason = format!( + "authority '{}' in node policy for '{}' not found in attribute providing services.", attr.authority(), node.name - ).as_str())); + ); + error!("from_yaml(): {reason}"); + return Err(serde_yaml::Error::custom(reason)); } } } @@ -238,10 +239,9 @@ output: let result = TemplateGraph::from_yaml(yaml_content); assert!(result.is_err()); let error = result.err().unwrap(); - assert!( - error - .to_string() - .contains("`authorities.attestation-services` must not be empty.") + assert_eq!( + error.to_string(), + "`authorities.attestation-services` must not be empty" ); } @@ -467,6 +467,9 @@ output: let result = TemplateGraph::from_yaml(yaml_content); assert!(result.is_err()); let error = result.err().unwrap(); - assert!(error.to_string().contains("Authority 'finra' in node policy for 'fetch_private' not found in attribute providing services.")); + assert_eq!( + error.to_string(), + "authority 'finra' in node policy for 'fetch_private' not found in attribute providing services." + ); } }