diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index a99495b..2664d3e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -36,7 +36,6 @@ jobs: check-path: "${{ matrix.path }}" unit-tests: name: "Build ${{ matrix.build_type }} with ${{ matrix.cxx }}" - needs: [ "codespell-check" , "format-check" ] runs-on: ubuntu-22.04 strategy: matrix: @@ -112,8 +111,9 @@ jobs: run: | pip install --upgrade gcovr gcovr \ - --exclude-throw-branches \ - --xml coverage.xml \ + --exclude-throw-branches \ + --exclude tests/ \ + --xml coverage.xml \ --gcov-executable "${{ startsWith(matrix.cxx, 'clang-') && format('llvm-cov-{0} gcov', env.CXX_VERSION) || format('gcov-{0}', env.CXX_VERSION) }}" \ ../build - name: "Upload coverage report" @@ -127,7 +127,7 @@ jobs: upload-to-codecov: name: "Codecov report upload" - needs: [ "unit-tests" ] + needs: [ "unit-tests" , "codespell-check" , "format-check" ] runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 diff --git a/CMakeLists.txt b/CMakeLists.txt index 78890f4..6f49486 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,15 +34,15 @@ if (NOT PROJECT_IS_TOP_LEVEL) message(STATUS "CAPIO-CL included as subproject → tests disabled") endif () -if(CMAKE_BUILD_TYPE STREQUAL "Debug") +if (CMAKE_BUILD_TYPE STREQUAL "Debug") add_compile_options(-O0 -g) -endif() +endif () -if(ENABLE_COVERAGE) +if (ENABLE_COVERAGE) message(STATUS "Building with code coverage instrumentation") add_compile_options(--coverage -O0 -g) add_link_options(--coverage) -endif() +endif () ##################################### # External projects @@ -142,6 +142,18 @@ if (CAPIO_CL_BUILD_TESTS) include(GoogleTest) gtest_discover_tests(CAPIO_CL_tests) + ##################################### + # Copy JSON test files + ##################################### + set(TEST_JSON_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/jsons") + + add_custom_command( + TARGET CAPIO_CL_tests PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "/tmp/capio_cl_jsons" + COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_JSON_DIR} "/tmp/capio_cl_jsons" + COMMENT "Copying JSON test files to build directory" + ) + ##################################### # Install rules ##################################### @@ -149,4 +161,10 @@ if (CAPIO_CL_BUILD_TESTS) install(TARGETS CAPIO_CL_tests RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) + + # Copy tests jsons + install(DIRECTORY ${TEST_JSON_DIR} + DESTINATION ${CMAKE_INSTALL_BINDIR}/jsons + FILES_MATCHING PATTERN "*.json" + ) endif () diff --git a/capiocl.hpp b/capiocl.hpp index 8459a1a..0dcf075 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -107,10 +107,13 @@ class Engine { * * @param str Input string. * @param n Number of characters to keep from the end. - * @return Truncated string with optional "[..]" prefix. + * @return Truncated string with "[..]" prefix. */ static std::string truncateLastN(const std::string &str, const std::size_t n) { - return str.length() > n ? "[..] " + str.substr(str.length() - n) : str; + if (str.length() > n) { + return "[..] " + str.substr(str.length() - n); + } + return str; } protected: @@ -320,7 +323,7 @@ class Engine { std::vector getConsumers(const std::string &path); /// @brief Get the commit-on-close counter for a file. - long getCommitCloseCount(std::filesystem::path::iterator::reference path) const; + long getCommitCloseCount(std::filesystem::path::iterator::reference path); /// @brief Get file dependencies. std::vector getCommitOnFileDependencies(const std::filesystem::path &path); @@ -350,7 +353,7 @@ class Engine { bool isConsumer(const std::string &path, const std::string &app_name); /** - * @brief Check if a file is firable. + * @brief Check if a file is firable, that is fire rule is no_update. * * @param path File path. * @return true if the file is firable, false otherwise. @@ -371,7 +374,7 @@ class Engine { * @param path File path. * @return true if excluded, false otherwise. */ - bool isExcluded(const std::string &path) const; + bool isExcluded(const std::string &path); /** * @brief Check if a path is a directory. @@ -396,6 +399,8 @@ class Engine { * @return */ bool isPermanent(const std::string &path); + + bool operator==(const capiocl::Engine &other) const; }; /** @@ -403,41 +408,38 @@ class Engine { * */ class Parser { - /** - * @brief Check if a string is a representation of a integer number - * - * @param s - * @return true - * @return false - */ - static bool isInteger(const std::string &s); - /** - * @brief compare two paths - * - * @param path - * @param base - * @return true if @p path is a subdirectory of base - * @return false otherwise - */ - static inline bool firstIsSubpathOfSecond(const std::filesystem::path &path, - const std::filesystem::path &base); + static std::filesystem::path resolve(std::filesystem::path path, + const std::filesystem::path &prefix); public: /** * @brief Perform the parsing of the capio_server configuration file * * @param source - * @param resolve_prexix + * @param resolve_prefix * @param store_only_in_memory Set to true to set all files to be stored in memory * @return Tuple with workflow name and CapioCLEngine instance with the information provided by * the config file */ static std::tuple parse(const std::filesystem::path &source, - std::filesystem::path &resolve_prexix, + const std::filesystem::path &resolve_prefix = "", bool store_only_in_memory = false); }; +/** + * Custom exception for errors occurring within the Parser component + */ +class ParserException : public std::exception { + std::string message; + + public: + explicit ParserException(const std::string &msg) : message(msg) { + print_message(CLI_LEVEL_ERROR, msg); + } + const char *what() const noexcept override { return message.c_str(); } +}; + class Serializer { public: /** @@ -447,7 +449,7 @@ class Serializer { * @param workflow_name Name of the current workflow * @param filename path of output file @param filename */ - static void dump(const Engine &engine, const std::string workflow_name, + static void dump(const Engine &engine, const std::string &workflow_name, const std::filesystem::path &filename); }; } // namespace capiocl diff --git a/src/Engine.cpp b/src/Engine.cpp index fe240d2..408af90 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -10,9 +10,7 @@ void capiocl::Engine::print() const { print_message(CLI_LEVEL_JSON, "Composition of expected CAPIO FS: "); // Table header lines - print_message(CLI_LEVEL_JSON, - "|============================================================================" - "==========================================================|"); + print_message(CLI_LEVEL_JSON, "*" + std::string(134, '=') + "*"); print_message(CLI_LEVEL_JSON, "|" + std::string(134, ' ') + "|"); @@ -34,9 +32,7 @@ void capiocl::Engine::print() const { "| \033[48;5;172m \033[0m File stored on file system" + std::string(77, ' ') + "|"); - print_message(CLI_LEVEL_JSON, - "|============================================================================" - "==========================================================|"); + print_message(CLI_LEVEL_JSON, "|" + std::string(134, '=') + "|"); print_message(CLI_LEVEL_JSON, "|======|===================|===================|====================|========" @@ -101,25 +97,22 @@ void capiocl::Engine::print() const { fire_rule = std::get<3>(itm.second); bool exclude = std::get<5>(itm.second), permanent = std::get<4>(itm.second); - line << " " << commit_rule << std::setfill(' ') - << std::setw(20 - commit_rule.length()) << " | " << fire_rule - << std::setfill(' ') << std::setw(13 - fire_rule.length()) << " | " - << " " << (permanent ? "YES" : "NO ") << " | " - << (exclude ? "YES" : "NO ") << " | " << n_files - << std::setw(10 - n_files.length()) << " |"; + line << " " << commit_rule << std::setfill(' '); + line << std::setw(20 - commit_rule.length()) << " | " << fire_rule; + line << std::setfill(' ') << std::setw(13 - fire_rule.length()) << " | "; + line << " " << (permanent ? "YES" : "NO ") << " | "; + line << (exclude ? "YES" : "NO "); + line << " | " << n_files << std::setw(10 - n_files.length()) << " |"; } else { - line << std::setfill(' ') << std::setw(20) << "|" << std::setfill(' ') - << std::setw(13) << "|" << std::setfill(' ') << std::setw(12) << "|" - << std::setfill(' ') << std::setw(10) << "|" << std::setw(11) << "|"; + line << std::setfill(' ') << std::setw(20) << "|" << std::setfill(' '); + line << std::setw(13) << "|" << std::setfill(' ') << std::setw(12) << "|"; + line << std::setfill(' ') << std::setw(10) << "|" << std::setw(11) << "|"; } print_message(CLI_LEVEL_JSON, line.str()); } - print_message(CLI_LEVEL_JSON, - "*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - "~~~~~~~~~~~~~~~~~~" - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*"); + print_message(CLI_LEVEL_JSON, "*" + std::string(134, '~') + "*"); } print_message(CLI_LEVEL_JSON, ""); @@ -128,7 +121,7 @@ void capiocl::Engine::print() const { bool capiocl::Engine::contains(const std::filesystem::path &file) { START_LOG(gettid(), "call(file=%s)", file.c_str()); for (auto &[fst, snd] : _locations) { - if (fnmatch(file.c_str(), fst.c_str(), FNM_PATHNAME) == 0) { + if (fnmatch(fst.c_str(), file.c_str(), FNM_PATHNAME) == 0) { return true; } } @@ -161,8 +154,8 @@ void capiocl::Engine::newFile(const std::string &path) { std::string matchKey; size_t matchSize = 0; for (const auto &[filename, data] : _locations) { - if (fnmatch(filename.c_str(), path.c_str(), FNM_PATHNAME) == 0 && - filename.length() > matchSize) { + const bool match = fnmatch(filename.c_str(), path.c_str(), FNM_PATHNAME) == 0; + if (match && filename.length() > matchSize) { LOG("Found match with %s", filename.c_str()); matchSize = filename.length(); matchKey = filename; @@ -203,40 +196,63 @@ long capiocl::Engine::getDirectoryFileCount(const std::string &path) { return std::get<8>(itm->second); } this->newFile(path); - return getDirectoryFileCount(path); } void capiocl::Engine::addProducer(const std::string &path, std::string &producer) { START_LOG(gettid(), "call(path=%s, producer=%s)", path.c_str(), producer.c_str()); producer.erase(remove_if(producer.begin(), producer.end(), isspace), producer.end()); - newFile(path); + if (const auto itm = _locations.find(path); itm != _locations.end()) { - std::get<0>(itm->second).emplace_back(producer); + if (auto vec = std::get<0>(itm->second); + std::find(vec.begin(), vec.end(), producer) == vec.end()) { + std::get<0>(itm->second).emplace_back(producer); + } + return; } + this->newFile(path); + this->addProducer(path, producer); } void capiocl::Engine::addConsumer(const std::string &path, std::string &consumer) { START_LOG(gettid(), "call(path=%s, consumer=%s)", path.c_str(), consumer.c_str()); consumer.erase(remove_if(consumer.begin(), consumer.end(), isspace), consumer.end()); if (const auto itm = _locations.find(path); itm != _locations.end()) { - std::get<1>(itm->second).emplace_back(consumer); + if (auto vec = std::get<1>(itm->second); + std::find(vec.begin(), vec.end(), consumer) == vec.end()) { + std::get<1>(itm->second).emplace_back(consumer); + } + return; } + this->newFile(path); + this->addConsumer(path, consumer); } + void capiocl::Engine::addFileDependency(const std::string &path, std::string &file_dependency) { START_LOG(gettid(), "call(path=%s, consumer=%s)", path.c_str(), consumer.c_str()); file_dependency.erase(remove_if(file_dependency.begin(), file_dependency.end(), isspace), file_dependency.end()); if (const auto itm = _locations.find(path); itm != _locations.end()) { - std::get<9>(itm->second).emplace_back(file_dependency); + if (auto vec = std::get<9>(itm->second); + std::find(vec.begin(), vec.end(), file_dependency) == vec.end()) { + std::get<9>(itm->second).emplace_back(file_dependency); + } + + return; } + this->newFile(path); + this->setCommitRule(path, capiocl::COMMITTED_ON_FILE); + this->addFileDependency(path, file_dependency); } void capiocl::Engine::setCommitRule(const std::string &path, const std::string &commit_rule) { START_LOG(gettid(), "call(path=%s, commit_rule=%s)", path.c_str(), commit_rule.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<2>(itm->second) = commit_rule; + return; } + this->newFile(path); + this->setCommitRule(path, commit_rule); } std::string capiocl::Engine::getCommitRule(const std::string &path) { @@ -269,7 +285,10 @@ void capiocl::Engine::setFireRule(const std::string &path, const std::string &fi START_LOG(gettid(), "call(path=%s, fire_rule=%s)", path.c_str(), fire_rule.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<3>(itm->second) = fire_rule; + return; } + this->newFile(path); + setFireRule(path, fire_rule); } bool capiocl::Engine::isFirable(const std::string &path) { @@ -289,7 +308,10 @@ void capiocl::Engine::setPermanent(const std::string &path, bool value) { START_LOG(gettid(), "call(path=%s, value=%s)", path.c_str(), value ? "true" : "false"); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<4>(itm->second) = value; + return; } + this->newFile(path); + setPermanent(path, value); } bool capiocl::Engine::isPermanent(const std::string &path) { @@ -306,21 +328,30 @@ void capiocl::Engine::setExclude(const std::string &path, const bool value) { START_LOG(gettid(), "call(path=%s, value=%s)", path.c_str(), value ? "true" : "false"); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<5>(itm->second) = value; + return; } + this->newFile(path); + setExclude(path, value); } void capiocl::Engine::setDirectory(const std::string &path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<6>(itm->second) = false; + return; } + this->newFile(path); + setDirectory(path); } void capiocl::Engine::setFile(const std::string &path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<6>(itm->second) = true; + return; } + this->newFile(path); + setFile(path); } bool capiocl::Engine::isFile(const std::string &path) { @@ -340,18 +371,28 @@ void capiocl::Engine::setCommitedCloseNumber(const std::string &path, const long START_LOG(gettid(), "call(path=%s, num=%ld)", path.c_str(), num); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<7>(itm->second) = num; + return; } + this->newFile(path); + setCommitedCloseNumber(path, num); } void capiocl::Engine::setDirectoryFileCount(const std::string &path, long num) { START_LOG(gettid(), "call(path=%s, num=%ld)", path.c_str(), num); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<8>(itm->second) = num; + return; } + this->newFile(path); + this->setDirectory(path); + this->setDirectoryFileCount(path, num); } void capiocl::Engine::remove(const std::string &path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); + if (const auto itm = _locations.find(path); itm == _locations.end()) { + return; + } _locations.erase(path); } @@ -381,21 +422,8 @@ bool capiocl::Engine::isConsumer(const std::string &path, const std::string &app } LOG("No exact match found in locations. checking for globs"); - // check for glob. Here we do not use the LMP check - for (const auto &[k, entry] : _locations) { - if (fnmatch(k.c_str(), path.c_str(), FNM_PATHNAME) == 0) { - LOG("Found possible glob match"); - std::vector producers = std::get<1>(entry); - DBG(gettid(), [&](const std::vector &arr) { - for (auto itm : arr) { - LOG("producer: %s", itm.c_str()); - } - }(producers)); - return std::find(producers.begin(), producers.end(), app_name) != producers.end(); - } - } - LOG("No match has been found"); - return false; + this->newFile(path); + return isConsumer(path, app_name); } std::vector capiocl::Engine::getProducers(const std::string &path) { @@ -403,7 +431,8 @@ std::vector capiocl::Engine::getProducers(const std::string &path) if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<0>(itm->second); } - return {}; + this->newFile(path); + return getProducers(path); } bool capiocl::Engine::isProducer(const std::string &path, const std::string &app_name) { @@ -424,21 +453,8 @@ bool capiocl::Engine::isProducer(const std::string &path, const std::string &app } LOG("No exact match found in locations. checking for globs"); - // check for glob. Here we do not use the LMP check - for (const auto &[k, entry] : _locations) { - if (fnmatch(k.c_str(), path.c_str(), FNM_PATHNAME) == 0) { - LOG("Found possible glob match"); - std::vector producers = std::get<0>(entry); - DBG(gettid(), [&](const std::vector &arr) { - for (auto itm : arr) { - LOG("producer: %s", itm.c_str()); - } - }(producers)); - return std::find(producers.begin(), producers.end(), app_name) != producers.end(); - } - } - LOG("No match has been found"); - return false; + this->newFile(path); + return isProducer(path, app_name); } void capiocl::Engine::setFileDeps(const std::filesystem::path &path, @@ -447,25 +463,32 @@ void capiocl::Engine::setFileDeps(const std::filesystem::path &path, if (dependencies.empty()) { return; } - if (_locations.find(path) == _locations.end()) { - this->newFile(path); - } - std::get<9>(_locations.at(path)) = dependencies; + for (const auto &itm : dependencies) { - LOG("Creating new fie (if it exists) for path %s", itm.c_str()); + LOG("Creating new file for path %s", itm.c_str()); newFile(itm); } + + if (_locations.find(path) != _locations.end()) { + std::get<9>(_locations.at(path)) = dependencies; + return; + } + this->newFile(path); + setFileDeps(path, dependencies); } -long capiocl::Engine::getCommitCloseCount(std::filesystem::path::iterator::reference path) const { +long capiocl::Engine::getCommitCloseCount(std::filesystem::path::iterator::reference path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); - long count = 0; + if (const auto itm = _locations.find(path); itm != _locations.end()) { - count = std::get<7>(itm->second); + auto count = std::get<7>(itm->second); + LOG("Expected number on close to commit file: %d", count); + return count; } - LOG("Expected number on close to commit file: %d", count); - return count; -}; + + this->newFile(path); + return getCommitCloseCount(path); +} std::vector capiocl::Engine::getCommitOnFileDependencies(const std::filesystem::path &path) { @@ -476,9 +499,14 @@ capiocl::Engine::getCommitOnFileDependencies(const std::filesystem::path &path) } void capiocl::Engine::setStoreFileInMemory(const std::filesystem::path &path) { + if (const auto itm = _locations.find(path); itm != _locations.end()) { + std::get<10>(_locations.at(path)) = true; + return; + } this->newFile(path); - std::get<10>(_locations.at(path)) = true; + setStoreFileInMemory(path); } + void capiocl::Engine::setAllStoreInMemory() { for (const auto &[fst, snd] : _locations) { this->setStoreFileInMemory(fst); @@ -486,15 +514,20 @@ void capiocl::Engine::setAllStoreInMemory() { } void capiocl::Engine::setStoreFileInFileSystem(const std::filesystem::path &path) { + if (const auto itm = _locations.find(path); itm != _locations.end()) { + std::get<10>(_locations.at(path)) = false; + return; + } this->newFile(path); - std::get<10>(_locations.at(path)) = false; + setStoreFileInFileSystem(path); } bool capiocl::Engine::isStoredInMemory(const std::filesystem::path &path) { if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<10>(itm->second); } - return false; + this->newFile(path); + return isStoredInMemory(path); } std::vector capiocl::Engine::getFileToStoreInMemory() { @@ -523,25 +556,104 @@ std::string capiocl::Engine::getHomeNode(const std::string &path) { return node_name; } -bool capiocl::Engine::isExcluded(const std::string &path) const { +bool capiocl::Engine::isExcluded(const std::string &path) { if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<5>(itm->second); } LOG("Checking against GLOB"); - size_t lpm_match_size = -1; - bool lpm_match = false; - for (const auto &[glob_path, entry] : _locations) { - if (fnmatch(glob_path.c_str(), path.c_str(), FNM_PATHNAME) == 0) { - LOG("Found match with %s", glob_path.c_str()); - if (glob_path.length() > lpm_match_size) { - lpm_match_size = glob_path.length(); - lpm_match = std::get<5>(entry); - LOG("Match is longer than previous match. storing value = %s", - lpm_match ? "true" : "false"); + this->newFile(path); + return isExcluded(path); +} + +bool capiocl::Engine::operator==(const capiocl::Engine &other) const { + const auto &other_locations = other.getLocations(); + + // check same size + if (this->_locations.size() != other_locations->size()) { + return false; + } + + // check same entry paths + for (const auto &[this_path, this_itm] : this->_locations) { + if (other_locations->find(this_path) == other_locations->end()) { + return false; + } + // check same config for each path + auto other_itm = other_locations->at(this_path); + + // check for "primitive" data types + if (std::get<2>(this_itm) != std::get<2>(other_itm)) { + return false; + } + + if (std::get<3>(this_itm) != std::get<3>(other_itm)) { + return false; + } + + if (std::get<4>(this_itm) != std::get<4>(other_itm)) { + return false; + } + + if (std::get<5>(this_itm) != std::get<5>(other_itm)) { + return false; + } + + if (std::get<6>(this_itm) != std::get<6>(other_itm)) { + return false; + } + + if (std::get<7>(this_itm) != std::get<7>(other_itm)) { + return false; + } + + if (std::get<8>(this_itm) != std::get<8>(other_itm)) { + return false; + } + + if (std::get<10>(this_itm) != std::get<10>(other_itm)) { + return false; + } + + // check for producer vector + auto this_producer = std::get<0>(this_itm); + auto other_producer = std::get<0>(other_itm); + if (this_producer.size() != other_producer.size()) { + return false; + } + for (const auto &entry : this_producer) { + if (std::find(other_producer.begin(), other_producer.end(), entry) == + other_producer.end()) { + return false; + } + } + + // check for consumer vector + auto this_consumer = std::get<1>(this_itm); + auto other_consumer = std::get<1>(other_itm); + if (this_consumer.size() != other_consumer.size()) { + return false; + } + for (const auto &entry : this_consumer) { + if (std::find(other_consumer.begin(), other_consumer.end(), entry) == + other_consumer.end()) { + return false; + } + } + + // check for file dependencies + + auto this_deps = std::get<9>(this_itm); + auto other_deps = std::get<9>(other_itm); + if (this_deps.size() != other_deps.size()) { + return false; + } + for (const auto &entry : this_deps) { + if (std::find(other_deps.begin(), other_deps.end(), entry) == other_deps.end()) { + return false; } } } - return lpm_match; -} + return true; +} \ No newline at end of file diff --git a/src/Parser.cpp b/src/Parser.cpp index 76daaf4..0440802 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -3,67 +3,50 @@ #include #include -bool capiocl::Parser::isInteger(const std::string &s) { - START_LOG(gettid(), "call(%s)", s.c_str()); - bool res = false; - if (!s.empty()) { - char *p; - strtol(s.c_str(), &p, 10); - res = *p == 0; +std::filesystem::path capiocl::Parser::resolve(std::filesystem::path path, + const std::filesystem::path &prefix) { + if (prefix.empty()) { + return path; } - return res; -}; -bool capiocl::Parser::firstIsSubpathOfSecond(const std::filesystem::path &path, - const std::filesystem::path &base) { - const auto mismatch_pair = std::mismatch(path.begin(), path.end(), base.begin(), base.end()); - return mismatch_pair.second == base.end(); + if (path.is_absolute()) { + return path; + } + + auto resolved = prefix / path; + const auto msg = "Path : " + path.string() + " IS RELATIVE! Resolved to: " + resolved.string(); + print_message(CLI_LEVEL_WARNING, msg); + + return resolved; } std::tuple -capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path &resolve_prefix, - bool store_only_in_memory) { +capiocl::Parser::parse(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto locations = new Engine(); START_LOG(gettid(), "call(config_file='%s')", source.c_str()); - if (resolve_prefix.empty()) { - resolve_prefix = "."; - } - - locations->newFile("*"); - locations->setDirectory("*"); - if (store_only_in_memory) { - locations->setStoreFileInMemory("*"); - } - if (source.empty()) { - return {workflow_name, locations}; + throw capiocl::ParserException("Empty source file name!"); } // ---- Load JSON ---- std::ifstream file(source); if (!file.is_open()) { - std::string msg = "Failed to open config file: " + source.string(); - print_message(CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); + throw capiocl::ParserException("Failed to open file!"); } nlohmann::json doc; - try { - file >> doc; - } catch (const std::exception &e) { - std::string err = "JSON parse error: "; - err += e.what(); - print_message(CLI_LEVEL_ERROR, err); - ERR_EXIT(err.c_str()); - } + file >> doc; // ---- workflow name ---- - if (!doc.contains("name") || !doc["name"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Missing workflow name!"); - ERR_EXIT("Error: workflow name is mandatory"); + if (!doc.contains("name")) { + throw capiocl::ParserException("Missing workflow name!"); + } + if (!doc["name"].is_string()) { + throw capiocl::ParserException("Wrong data type for workflow name!"); } workflow_name = doc["name"].get(); @@ -71,14 +54,19 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat LOG("Parsing configuration for workflow: %s", workflow_name.c_str()); // ---- IO_Graph ---- - if (!doc.contains("IO_Graph") || !doc["IO_Graph"].is_array()) { - ERR_EXIT("Error: IO_Graph section missing or invalid"); + if (!doc.contains("IO_Graph")) { + throw capiocl::ParserException("Missing IO_Graph section"); + } + if (!doc["IO_Graph"].is_array()) { + throw capiocl::ParserException("Wrong data type for IO_Graph section"); } for (const auto &app : doc["IO_Graph"]) { - if (!app.contains("name") || !app["name"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Missing IO_Graph name or name is not a valid string!"); - ERR_EXIT("Error: app name is mandatory"); + if (!app.contains("name")) { + throw capiocl::ParserException("Missing name for streaming item!"); + } + if (!app["name"].is_string()) { + throw capiocl::ParserException("Wrong type for name streaming entry!"); } std::string app_name = app["name"].get(); @@ -86,38 +74,33 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat LOG("Parsing config for app %s", app_name.c_str()); // ---- input_stream ---- - if (!app.contains("input_stream") || !app["input_stream"].is_array()) { - std::string msg = "No input_stream section found for app " + app_name; - print_message(CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); + if (!app.contains("input_stream")) { + throw capiocl::ParserException("No input_stream section found for app " + app_name); + } + if (!app["input_stream"].is_array()) { + throw capiocl::ParserException("input_stream section for app " + app_name + + " is not array!"); } print_message(CLI_LEVEL_JSON, "Parsing input_stream for app " + app_name); for (const auto &itm : app["input_stream"]) { - std::filesystem::path file_path(itm.get()); - if (file_path.is_relative()) { - print_message(CLI_LEVEL_WARNING, - "Path : " + file_path.string() + " IS RELATIVE! resolving..."); - file_path = resolve_prefix / file_path; - } + auto file_path = resolve(itm.get(), resolve_prefix); locations->newFile(file_path); locations->addConsumer(file_path, app_name); } // ---- output_stream ---- - if (!app.contains("output_stream") || !app["output_stream"].is_array()) { - std::string msg = "No output_stream section found for app " + app_name; - print_message(CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); + if (!app.contains("output_stream")) { + throw capiocl::ParserException("No output_stream section found for app " + app_name); + } + if (!app["output_stream"].is_array()) { + throw capiocl::ParserException("output_stream section for app " + app_name + + " is not array!"); } + print_message(CLI_LEVEL_JSON, "Parsing output_stream for app " + app_name); for (const auto &itm : app["output_stream"]) { - std::filesystem::path file_path(itm.get()); - if (file_path.is_relative()) { - print_message(CLI_LEVEL_WARNING, - "Path : " + file_path.string() + " IS RELATIVE! resolving..."); - file_path = resolve_prefix / file_path; - } + auto file_path = resolve(itm.get(), resolve_prefix); locations->newFile(file_path); locations->addProducer(file_path, app_name); } @@ -130,40 +113,31 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat std::vector streaming_names; std::vector file_deps; std::string commit_rule = COMMITTED_ON_TERMINATION, mode = MODE_UPDATE; - long int n_close = -1; - int64_t n_files = -1; + long int n_close = 0; + int64_t n_files = 0; // name or dirname if (stream_item.contains("name") && stream_item["name"].is_array()) { for (const auto &nm : stream_item["name"]) { - std::filesystem::path p(nm.get()); - if (p.is_relative()) { - p = resolve_prefix / p; - } - streaming_names.push_back(p); + auto nm_resolved = resolve(nm.get(), resolve_prefix); + streaming_names.push_back(nm_resolved); } } else if (stream_item.contains("dirname") && stream_item["dirname"].is_array()) { is_file = false; for (const auto &nm : stream_item["dirname"]) { - std::filesystem::path p(nm.get()); - if (p.is_relative()) { - p = resolve_prefix / p; - } - streaming_names.push_back(p); + auto nm_resolved = resolve(nm.get(), resolve_prefix); + streaming_names.push_back(nm_resolved); } } else { - print_message( - CLI_LEVEL_ERROR, + throw capiocl::ParserException( "Missing streaming name/dirname, or name/dirname is not an array for app " + - app_name); - ERR_EXIT("error: either name or dirname in streaming section is required"); + app_name); } // Commit rule. Optional in nature, hence no check required! if (stream_item.contains("committed")) { if (!stream_item["committed"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Error: invalid type for commit rule!"); - ERR_EXIT("Error: invalid type for commit rule!"); + throw capiocl::ParserException("Error: invalid type for commit rule!"); } std::string committed = stream_item["committed"].get(); @@ -171,15 +145,20 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat if (pos != std::string::npos) { commit_rule = committed.substr(0, pos); std::string count_str = committed.substr(pos + 1); - if (!isInteger(count_str)) { - ERR_EXIT("invalid number in commit rule"); + try { + size_t num_len; + std::stoi(count_str, &num_len); + } catch (...) { + auto msg = "commit rule argument is not an integer!"; + throw capiocl::ParserException(msg); } + if (commit_rule == COMMITTED_ON_CLOSE) { n_close = std::stol(count_str); } else if (commit_rule == COMMITTED_N_FILES) { n_files = std::stol(count_str); } else { - ERR_EXIT("invalid commit rule type"); + throw capiocl::ParserException("Invalid commit rule!"); } } else { commit_rule = committed; @@ -189,14 +168,12 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat if (commit_rule == COMMITTED_ON_FILE) { if (!stream_item.contains("file_deps") || !stream_item["file_deps"].is_array()) { - ERR_EXIT("commit rule is on_file but no file_deps section found"); + throw capiocl::ParserException( + "commit rule is on_file but no file_deps section found"); } for (const auto &dep : stream_item["file_deps"]) { - std::filesystem::path p(dep.get()); - if (p.is_relative()) { - p = resolve_prefix / p; - } - file_deps.push_back(p); + auto dep_resolved = resolve(dep.get(), resolve_prefix); + file_deps.push_back(dep_resolved); } } @@ -205,35 +182,34 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat commit_rule != capiocl::COMMITTED_ON_CLOSE && commit_rule != capiocl::COMMITTED_ON_FILE && commit_rule != capiocl::COMMITTED_ON_TERMINATION) { - print_message(CLI_LEVEL_ERROR, "Error: commit rule " + commit_rule + - " is not one of the allowed one!"); - ERR_EXIT("Unknown commit rule %s", commit_rule.c_str()); + throw capiocl::ParserException("Error: commit rule " + commit_rule + + " is not one of the allowed one!"); } } // Firing rule. Optional in nature, hence no check required! if (stream_item.contains("mode")) { if (!stream_item["mode"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Error: invalid type for mode"); - ERR_EXIT("Error: invalid type for mode"); + throw capiocl::ParserException("Error: invalid firing rule data type"); } mode = stream_item["mode"].get(); if (mode != capiocl::MODE_UPDATE && mode != capiocl::MODE_NO_UPDATE) { - print_message(CLI_LEVEL_ERROR, - "Error: invalid firing rule provided for app: " + app_name); - ERR_EXIT("Error: invalid firing rule provided for app: %s", - app_name.c_str()); + throw capiocl::ParserException( + "Error: fire rule is not one of the allowed ones for app: " + app_name); } } // n_files (optional) - if (stream_item.contains("n_files") && stream_item["n_files"].is_number_integer()) { + if (stream_item.contains("n_files")) { + if (!stream_item["n_files"].is_number_integer()) { + throw capiocl::ParserException("wrong type for n_files!"); + } n_files = stream_item["n_files"].get(); } for (auto &path : streaming_names) { - if (n_files != -1) { + if (n_files != 0) { locations->setDirectoryFileCount(path, n_files); } if (is_file) { @@ -283,17 +259,26 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat } // ---- storage ---- - if (doc.contains("storage") && doc["storage"].is_object()) { + if (doc.contains("storage")) { + if (!doc["storage"].is_object()) { + throw capiocl::ParserException("Wrong data type for storage section!"); + } const auto &storage = doc["storage"]; - if (storage.contains("memory") && storage["memory"].is_array()) { + if (storage.contains("memory")) { + if (!storage["memory"].is_array()) { + throw capiocl::ParserException("Wrong data type for memory storage section"); + } for (const auto &f : storage["memory"]) { auto file_str = f.get(); locations->setStoreFileInMemory(file_str); } } - if (storage.contains("fs") && storage["fs"].is_array()) { + if (storage.contains("fs")) { + if (!storage["fs"].is_array()) { + throw capiocl::ParserException("Wrong data type for fs storage section"); + } for (const auto &f : storage["fs"]) { auto file_str = f.get(); locations->setStoreFileInFileSystem(file_str); diff --git a/src/Serializer.cpp b/src/Serializer.cpp index 89d989b..d12704c 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -3,39 +3,30 @@ #include #include -void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string workflow_name, +void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string &workflow_name, const std::filesystem::path &filename) { START_LOG(gettid(), "call(output='%s')", target.c_str()); nlohmann::json doc; doc["name"] = workflow_name; - // Retrieve the files map - const auto *files = engine.getLocations(); // adjust if it's a different getter + const auto *files = engine.getLocations(); - // Build helper maps: app → inputs / outputs std::unordered_map> app_inputs; std::unordered_map> app_outputs; - // For permanent/exclude/storage std::vector permanent; std::vector exclude; std::vector memory_storage; std::vector fs_storage; + nlohmann::json storage; nlohmann::json io_graph = nlohmann::json::array(); - // We'll also need a mapping of each file → its metadata for streaming reconstruction for (const auto &[path, data] : *files) { const auto &[producers, consumers, commit_rule, fire_rule, permanent_flag, excluded_flag, is_file, n_close, n_dir_files, file_deps, store_in_memory] = data; - if (path == "*") { - LOG("Skipping * path"); - print_message(CLI_LEVEL_WARNING, "Skipping * path"); - continue; - } - // Collect permanent/exclude info if (permanent_flag) { permanent.push_back(path); } @@ -43,14 +34,8 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string exclude.push_back(path); } - // Collect storage info - if (store_in_memory) { - memory_storage.push_back(path); - } else { - fs_storage.push_back(path); - } + store_in_memory ? memory_storage.push_back(path) : fs_storage.push_back(path); - // Collect app relationships for (const auto &p : producers) { app_outputs[p].push_back(path); } @@ -59,20 +44,8 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string } } - // Construct IO_Graph section for (const auto &[app_name, outputs] : app_outputs) { nlohmann::json app; - app["name"] = app_name; - - if (app_inputs.count(app_name)) { - app["input_stream"] = app_inputs[app_name]; - } else { - app["input_stream"] = nlohmann::json::array(); - } - - app["output_stream"] = outputs; - - // ---- streaming ---- nlohmann::json streaming = nlohmann::json::array(); for (const auto &path : outputs) { @@ -81,83 +54,62 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string excluded_flag, is_file, n_close, n_dir_files, file_deps, store_in_fs] = data; - if (path == "*") { - LOG("Skipping * path"); - print_message(CLI_LEVEL_WARNING, "Skipping * path"); - continue; - } - - nlohmann::json sitem; - if (is_file) { - sitem["name"] = nlohmann::json::array({path}); - } else { - sitem["dirname"] = nlohmann::json::array({path}); - } + nlohmann::json streaming_item; + std::string committed = commit_rule; + const auto name_kind = is_file ? "name" : "dirname"; + streaming_item[name_kind] = {path}; // Commit rule - std::string committed = commit_rule; if (commit_rule == capiocl::COMMITTED_ON_CLOSE && n_close > 0) { committed += ":" + std::to_string(n_close); - } else if (commit_rule == capiocl::COMMITTED_N_FILES && n_dir_files > 0) { - committed += ":" + std::to_string(n_dir_files); } - - sitem["committed"] = committed; - - // Mode / fire rule - sitem["mode"] = fire_rule; - - // File dependencies (for COMMITTED_ON_FILE) - if (commit_rule == capiocl::COMMITTED_ON_FILE && !file_deps.empty()) { - sitem["file_deps"] = file_deps; + if (commit_rule == capiocl::COMMITTED_N_FILES && n_dir_files > 0) { + committed += ":" + std::to_string(n_dir_files); } - // Directory file count - if (n_dir_files > 0) { - sitem["n_files"] = n_dir_files; - } + streaming_item["file_deps"] = file_deps; + streaming_item["committed"] = committed; + streaming_item["mode"] = fire_rule; + streaming_item["n_files"] = n_dir_files; - streaming.push_back(sitem); + streaming.push_back(streaming_item); } - if (!streaming.empty()) { - app["streaming"] = streaming; - } + app["name"] = app_name; + app["input_stream"] = app_inputs[app_name]; + app["output_stream"] = outputs; + app["streaming"] = streaming; io_graph.push_back(app); } - doc["IO_Graph"] = io_graph; - - // ---- permanent / exclude ---- - if (!permanent.empty()) { - doc["permanent"] = permanent; - } - if (!exclude.empty()) { - doc["exclude"] = exclude; + /// Check for app names that have a output_stream empty + for (const auto &[app_name, inputs] : app_inputs) { + bool contained = false; + for (const auto &entry : io_graph) { + contained = entry["name"] == app_name; + if (contained) { + break; + } + } + if (!contained) { + nlohmann::json app; + app["name"] = app_name; + app["input_stream"] = inputs; + app["output_stream"] = nlohmann::json::array(); + io_graph.push_back(app); + } } - // ---- storage ---- - nlohmann::json storage; - if (!memory_storage.empty()) { - storage["memory"] = memory_storage; - } - if (!fs_storage.empty()) { - storage["fs"] = fs_storage; - } - if (!storage.empty()) { - doc["storage"] = storage; - } + doc["IO_Graph"] = io_graph; + doc["permanent"] = permanent; + doc["exclude"] = exclude; + storage["memory"] = memory_storage; + storage["fs"] = fs_storage; + doc["storage"] = storage; - // ---- Write JSON ---- std::ofstream out(filename); - if (!out.is_open()) { - std::string msg = "Failed to open output file: " + filename.string(); - capiocl::print_message(capiocl::CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); - } - - out << std::setw(4) << doc << std::endl; + out << std::setw(2) << doc << std::endl; capiocl::print_message(capiocl::CLI_LEVEL_INFO, "Configuration serialized to " + filename.string()); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index ee571fe..0b2af72 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -1,9 +1,11 @@ #include "capiocl.hpp" +#include #include TEST(testCapioClEngine, testInstantiation) { capiocl::Engine engine; EXPECT_EQ(engine.size(), 0); + engine.print(); } TEST(testCapioClEngine, testAddFileDefault) { @@ -76,6 +78,8 @@ TEST(testCapioClEngine, testAddFileManually) { EXPECT_FALSE(engine.isDirectory("test.dat")); EXPECT_EQ(engine.getDirectoryFileCount("test.dat"), 0); EXPECT_FALSE(engine.isStoredInMemory("test.dat")); + + engine.setFile("test.txt"); } TEST(testCapioClEngine, testAddFileManuallyGlob) { @@ -128,6 +132,9 @@ TEST(testCapioClEngine, testAddFileManuallyQuestion) { EXPECT_TRUE(engine.isDirectory("test.9")); EXPECT_EQ(engine.getDirectoryFileCount("test.a"), 10); EXPECT_FALSE(engine.isStoredInMemory("test.b")); + + engine.setDirectoryFileCount("myDir", 10); + EXPECT_EQ(engine.getDirectoryFileCount("myDir"), 10); } TEST(testCapioClEngine, testAddFileManuallyGlobExplcit) { @@ -185,6 +192,13 @@ TEST(testCapioClEngine, testProducerConsumersFileDependencies) { engine.addFileDependency("test.dat", file_dependencies[1]); EXPECT_EQ(engine.getCommitOnFileDependencies("test.dat").size(), 2); EXPECT_TRUE(engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1]); + + EXPECT_TRUE(engine.getCommitOnFileDependencies("myNewFile").empty()); + + engine.addFileDependency("myFile.txt", file_dependencies[0]); + EXPECT_TRUE(engine.getCommitRule("myFile.txt") == capiocl::COMMITTED_ON_FILE); + EXPECT_EQ(engine.getCommitOnFileDependencies("myFile.txt").size(), 1); + EXPECT_TRUE(engine.getCommitOnFileDependencies("myFile.txt")[0] == file_dependencies[0]); } TEST(testCapioClEngine, testProducerConsumersFileDependenciesGlob) { @@ -218,4 +232,407 @@ TEST(testCapioClEngine, testProducerConsumersFileDependenciesGlob) { engine.addFileDependency("test.dat", file_dependencies[1]); EXPECT_EQ(engine.getCommitOnFileDependencies("test.dat").size(), 2); EXPECT_TRUE(engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1]); +} + +TEST(testCapioClEngine, testCommitFirePermanentExcludeOnGlobs) { + capiocl::Engine engine; + engine.newFile("test.*"); + engine.setFireRule("test.*", capiocl::MODE_NO_UPDATE); + + EXPECT_TRUE(engine.isFirable("test.a")); + EXPECT_FALSE(engine.isFirable("testb")); + + engine.setCommitRule("testb", capiocl::COMMITTED_ON_FILE); + engine.setFileDeps("testb", {"test.a"}); + + engine.setPermanent("myFile", true); + EXPECT_TRUE(engine.isPermanent("myFile")); + EXPECT_FALSE(engine.isFirable("myFile")); + + EXPECT_FALSE(engine.isExcluded("testb")); + engine.setExclude("testb", true); + EXPECT_TRUE(engine.isExcluded("testb")); + engine.setExclude("testb", true); + EXPECT_TRUE(engine.isExcluded("testb")); + engine.setExclude("myFile.*", true); + EXPECT_TRUE(engine.isExcluded("myFile.txt")); + EXPECT_TRUE(engine.isExcluded("myFile.dat")); + + engine.setFireRule("test.c", capiocl::MODE_NO_UPDATE); + EXPECT_TRUE(engine.isFirable("test.c")); + + engine.setCommitedCloseNumber("test.e", 100); +} + +TEST(testCapioClEngine, testIsFileIsDirectoryGlob) { + capiocl::Engine engine; + engine.newFile("test.*"); + engine.setDirectory("test.d/"); + engine.setDirectory("test.d/bin/lib"); + EXPECT_TRUE(engine.isDirectory("test.d/bin/lib")); + EXPECT_TRUE(engine.isDirectory("test.d/")); + EXPECT_FALSE(engine.isDirectory("test.*")); +} + +TEST(testCapioClEngine, testAddRemoveFile) { + capiocl::Engine engine; + engine.newFile("test.*"); + EXPECT_TRUE(engine.contains("test.*")); + EXPECT_TRUE(engine.contains("test.txt")); + engine.remove("test.*"); + EXPECT_FALSE(engine.contains("test.*")); + engine.remove("data"); + EXPECT_FALSE(engine.contains("data")); +} + +TEST(testCapioClEngine, testProducersConsumers) { + capiocl::Engine engine; + engine.newFile("test.*"); + + std::string consumer = "consumer"; + std::string producer = "producer"; + + engine.addConsumer("test.txt", consumer); + engine.addProducer("test.txt.1", producer); + + EXPECT_TRUE(engine.isProducer("test.txt.1", producer)); + EXPECT_FALSE(engine.isProducer("test.txt.1", consumer)); + + EXPECT_FALSE(engine.isConsumer("test.txt", producer)); + EXPECT_TRUE(engine.isConsumer("test.txt", consumer)); + + engine.addConsumer("test.*", consumer); + engine.addProducer("test.*", producer); + + EXPECT_TRUE(engine.isProducer("test.*", producer)); + EXPECT_FALSE(engine.isProducer("test.*", consumer)); + EXPECT_TRUE(engine.isConsumer("test.*", consumer)); + EXPECT_FALSE(engine.isConsumer("test.*", producer)); + + engine.addProducer("test.k", producer); + engine.addConsumer("test.k", consumer); + EXPECT_TRUE(engine.isProducer("test.k", producer)); + EXPECT_TRUE(engine.isConsumer("test.k", consumer)); + + EXPECT_TRUE(engine.isConsumer("test.txt.2", consumer)); + EXPECT_FALSE(engine.isProducer("test.txt.3", consumer)); + EXPECT_FALSE(engine.isConsumer("test.txt.4", producer)); + EXPECT_TRUE(engine.isProducer("test.txt.4", producer)); + + EXPECT_EQ(engine.getProducers("myNewFile").size(), 0); + EXPECT_EQ(engine.getProducers("test.k").size(), 1); +} + +TEST(testCapioClEngine, testCommitCloseCount) { + capiocl::Engine engine; + engine.newFile("test.*"); + engine.setCommitRule("test.*", capiocl::COMMITTED_ON_CLOSE); + engine.setCommitedCloseNumber("test.e", 100); + + EXPECT_EQ(engine.getCommitCloseCount("test.e"), 100); + EXPECT_EQ(engine.getCommitCloseCount("test.d"), 0); + + engine.setCommitedCloseNumber("test.*", 30); + EXPECT_EQ(engine.getCommitCloseCount("test.f"), 30); + + engine.setCommitRule("myFile", capiocl::COMMITTED_ON_FILE); + EXPECT_TRUE(engine.getCommitRule("myFile") == capiocl::COMMITTED_ON_FILE); +} + +TEST(testCapioClEngine, testStorageOptions) { + capiocl::Engine engine; + engine.newFile("A"); + engine.newFile("B"); + + engine.setStoreFileInMemory("A"); + EXPECT_TRUE(engine.isStoredInMemory("A")); + EXPECT_FALSE(engine.isStoredInMemory("B")); + + engine.setStoreFileInMemory("B"); + EXPECT_TRUE(engine.isStoredInMemory("A")); + EXPECT_TRUE(engine.isStoredInMemory("B")); + + engine.setStoreFileInMemory("C"); + EXPECT_TRUE(engine.isStoredInMemory("C")); + + engine.newFile("D"); + EXPECT_FALSE(engine.isStoredInMemory("D")); + + engine.setAllStoreInMemory(); + EXPECT_TRUE(engine.isStoredInMemory("A")); + EXPECT_TRUE(engine.isStoredInMemory("B")); + EXPECT_TRUE(engine.isStoredInMemory("C")); + EXPECT_TRUE(engine.isStoredInMemory("D")); + + EXPECT_EQ(engine.getFileToStoreInMemory().size(), 4); + + engine.setStoreFileInFileSystem("A"); + engine.setStoreFileInFileSystem("B"); + engine.setStoreFileInFileSystem("C"); + engine.setStoreFileInFileSystem("D"); + EXPECT_FALSE(engine.isStoredInMemory("A")); + EXPECT_FALSE(engine.isStoredInMemory("B")); + EXPECT_FALSE(engine.isStoredInMemory("C")); + EXPECT_FALSE(engine.isStoredInMemory("D")); + + engine.setStoreFileInFileSystem("F"); + EXPECT_FALSE(engine.isStoredInMemory("F")); +} + +TEST(testCapioClEngine, testHomeNode) { + std::string nodename; + nodename.reserve(1024); + gethostname(nodename.data(), nodename.size()); + + capiocl::Engine engine; + engine.newFile("A"); + EXPECT_TRUE(engine.getHomeNode("A") == nodename); + EXPECT_TRUE(engine.getHomeNode("B") == nodename); +} + +TEST(testCapioClEngine, testInsertFileDependencies) { + capiocl::Engine engine; + + engine.setFileDeps("myFile.txt", {}); + engine.setFileDeps("test.txt", {"a", "b", "c"}); + EXPECT_EQ(engine.getCommitOnFileDependencies("test.txt").size(), 3); + EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[0] == "a"); + EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[1] == "b"); + EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[2] == "c"); +} + +TEST(testCapioClEngine, testEqualDifferentOperator) { + capiocl::Engine engine1, engine2; + + engine1.newFile("A"); + engine2.newFile("A"); + + engine1.setCommitRule("A", capiocl::COMMITTED_ON_CLOSE); + engine2.setCommitRule("A", capiocl::COMMITTED_ON_TERMINATION); + EXPECT_FALSE(engine1 == engine2); + engine2.setCommitRule("A", engine1.getCommitRule("A")); + + engine1.setFireRule("A", capiocl::MODE_NO_UPDATE); + engine2.setFireRule("A", capiocl::MODE_UPDATE); + EXPECT_FALSE(engine1 == engine2); + engine2.setFireRule("A", engine1.getFireRule("A")); + + engine1.setPermanent("A", true); + engine2.setPermanent("A", false); + EXPECT_FALSE(engine1 == engine2); + engine2.setPermanent("A", engine1.isPermanent("A")); + + engine1.setExclude("A", true); + engine2.setExclude("A", false); + EXPECT_FALSE(engine1 == engine2); + engine2.setExclude("A", engine1.isExcluded("A")); + + engine1.setFile("A"); + engine2.setDirectory("A"); + EXPECT_FALSE(engine1 == engine2); + engine2.setFile("A"); + + engine1.setCommitedCloseNumber("A", 10); + engine2.setCommitedCloseNumber("A", 5); + EXPECT_FALSE(engine1 == engine2); + engine2.setCommitedCloseNumber("A", engine1.getCommitCloseCount("A")); + + engine1.setDirectoryFileCount("A", 10); + engine2.setDirectoryFileCount("A", 5); + EXPECT_FALSE(engine1 == engine2); + engine2.setDirectoryFileCount("A", engine1.getDirectoryFileCount("A")); + + engine1.setStoreFileInFileSystem("A"); + engine2.setStoreFileInMemory("A"); + EXPECT_FALSE(engine1 == engine2); + engine2.setStoreFileInFileSystem("A"); + + engine2.newFile("C"); + EXPECT_FALSE(engine1 == engine2); +} + +TEST(testCapioClEngine, testEqualDifferentOperatorDifferentPathSameSize) { + capiocl::Engine engine1, engine2; + engine1.newFile("A"); + engine2.newFile("B"); + EXPECT_FALSE(engine1 == engine2); +} + +TEST(testCapioClEngine, testEqualDifferentProducers) { + capiocl::Engine engine1, engine2; + engine1.newFile("A"); + engine2.newFile("A"); + std::string stepA = "prod1"; + std::string stepB = "prod2"; + engine1.addProducer("A", stepA); + EXPECT_FALSE(engine1 == engine2); + engine2.addProducer("A", stepB); + EXPECT_FALSE(engine1 == engine2); + + engine1.addProducer("A", stepB); + engine2.addProducer("A", stepA); + + engine1.addConsumer("A", stepA); + EXPECT_FALSE(engine1 == engine2); + engine2.addConsumer("A", stepB); + EXPECT_FALSE(engine1 == engine2); +} + +TEST(testCapioClEngine, testFileDependenciesDifferences) { + capiocl::Engine engine1, engine2; + engine1.newFile("A"); + engine2.newFile("A"); + std::string depsA = "prod1"; + std::string depsB = "prod2"; + engine1.addFileDependency("A", depsA); + EXPECT_FALSE(engine1 == engine2); + engine2.addFileDependency("A", depsB); + EXPECT_FALSE(engine1 == engine2); + + engine1.addFileDependency("A", depsB); + EXPECT_FALSE(engine1 == engine2); + engine2.addFileDependency("A", depsA); + EXPECT_TRUE(engine1 == engine2); +} + +TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "my_command_history.txt", file_4_name = "/tmp"; + std::string producer_name = "_first", consumer_name = "_last", intermediate_name = "_middle"; + + capiocl::Engine engine; + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, intermediate_name); + engine.addProducer(file_2_name, intermediate_name); + engine.addConsumer(file_2_name, consumer_name); + engine.addConsumer(file_1_name, consumer_name); + + engine.setStoreFileInMemory(file_1_name); + engine.setCommitRule(file_1_name, capiocl::COMMITTED_ON_CLOSE); + engine.setCommitedCloseNumber(file_1_name, 3); + engine.setFireRule(file_1_name, capiocl::MODE_UPDATE); + engine.setPermanent(file_1_name, true); + + engine.setStoreFileInFileSystem(file_2_name); + engine.setCommitRule(file_2_name, capiocl::COMMITTED_ON_TERMINATION); + engine.setFireRule(file_1_name, capiocl::MODE_NO_UPDATE); + + engine.addProducer(file_3_name, producer_name); + engine.addProducer(file_3_name, consumer_name); + engine.addProducer(file_3_name, intermediate_name); + engine.setExclude(file_3_name, true); + + engine.setCommitRule(file_4_name, capiocl::COMMITTED_N_FILES); + engine.setFireRule(file_4_name, capiocl::MODE_NO_UPDATE); + engine.setDirectoryFileCount(file_4_name, 10); + engine.addProducer(file_4_name, intermediate_name); + + engine.print(); + + capiocl::Serializer::dump(engine, workflow_name, path); + + std::filesystem::path resolve = ""; + auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); + + EXPECT_TRUE(wf_name == workflow_name); + capiocl::print_message("", ""); + EXPECT_TRUE(engine == *new_engine); + + auto [wf_name1, new_engine1] = capiocl::Parser::parse(path, resolve, true); + EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); + + std::filesystem::remove(path); +} + +TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1NcloseNfiles) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt"; + std::string producer_name = "_first", consumer_name = "_last"; + + capiocl::Engine engine; + + engine.setDirectory(file_1_name); + engine.setDirectoryFileCount(file_1_name, 10); + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, consumer_name); + + capiocl::Serializer::dump(engine, workflow_name, path); + + std::filesystem::path resolve = ""; + auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); + + EXPECT_TRUE(wf_name == workflow_name); + capiocl::print_message("", ""); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); +} + +TEST(testCapioSerializerParser, testParserResolveAbsolute) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test0.json"); + auto [wf_name, engine] = capiocl::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(wf_name == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + EXPECT_TRUE(engine->contains("/tmp/file2")); + EXPECT_TRUE(engine->contains("/tmp/file3")); +} + +template std::string demangled_name(const T &obj) { + int status; + const char *mangled = typeid(obj).name(); + std::unique_ptr demangled( + abi::__cxa_demangle(mangled, nullptr, nullptr, &status), std::free); + return status == 0 ? demangled.get() : mangled; +} + +TEST(testCapioSerializerParser, testParserException) { + std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons"; + capiocl::print_message(capiocl::CLI_LEVEL_INFO, "Loading jsons from " + JSON_DIR.string()); + bool exception_catched = false; + + std::vector test_filenames = { + "", + "ANonExistingFile", + JSON_DIR / "V1_test1.json", + JSON_DIR / "V1_test2.json", + JSON_DIR / "V1_test3.json", + JSON_DIR / "V1_test4.json", + JSON_DIR / "V1_test5.json", + JSON_DIR / "V1_test6.json", + JSON_DIR / "V1_test7.json", + JSON_DIR / "V1_test8.json", + JSON_DIR / "V1_test9.json", + JSON_DIR / "V1_test10.json", + JSON_DIR / "V1_test11.json", + JSON_DIR / "V1_test12.json", + JSON_DIR / "V1_test13.json", + JSON_DIR / "V1_test14.json", + JSON_DIR / "V1_test15.json", + JSON_DIR / "V1_test16.json", + JSON_DIR / "V1_test17.json", + JSON_DIR / "V1_test18.json", + JSON_DIR / "V1_test19.json", + JSON_DIR / "V1_test20.json", + JSON_DIR / "V1_test21.json", + JSON_DIR / "V1_test22.json", + JSON_DIR / "V1_test23.json", + }; + + for (const auto &test : test_filenames) { + exception_catched = false; + try { + capiocl::print_message(capiocl::CLI_LEVEL_WARNING, "Testing on file " + test.string()); + auto [wf_name, engine] = capiocl::Parser::parse(test); + } catch (std::exception &e) { + exception_catched = true; + EXPECT_TRUE(demangled_name(e) == "capiocl::ParserException"); + EXPECT_GT(std::string(e.what()).size(), 0); + } + EXPECT_TRUE(exception_catched); + capiocl::print_message(capiocl::CLI_LEVEL_WARNING); + } } \ No newline at end of file diff --git a/tests/jsons/V1_test0.json b/tests/jsons/V1_test0.json new file mode 100644 index 0000000..e6ed298 --- /dev/null +++ b/tests/jsons/V1_test0.json @@ -0,0 +1,25 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "/tmp/file", + "file3" + ], + "output_stream": [ + "file1", + "file2" + ], + "streaming": [ + { + "name": [ + "/tmp/file" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test1.json b/tests/jsons/V1_test1.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/tests/jsons/V1_test1.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tests/jsons/V1_test10.json b/tests/jsons/V1_test10.json new file mode 100644 index 0000000..ec49bcf --- /dev/null +++ b/tests/jsons/V1_test10.json @@ -0,0 +1,12 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test11.json b/tests/jsons/V1_test11.json new file mode 100644 index 0000000..81a1b16 --- /dev/null +++ b/tests/jsons/V1_test11.json @@ -0,0 +1,17 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + {} + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test12.json b/tests/jsons/V1_test12.json new file mode 100644 index 0000000..a0228e7 --- /dev/null +++ b/tests/jsons/V1_test12.json @@ -0,0 +1,19 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test13.json b/tests/jsons/V1_test13.json new file mode 100644 index 0000000..4f17be1 --- /dev/null +++ b/tests/jsons/V1_test13.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test14.json b/tests/jsons/V1_test14.json new file mode 100644 index 0000000..28cd8a2 --- /dev/null +++ b/tests/jsons/V1_test14.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "a:b" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test15.json b/tests/jsons/V1_test15.json new file mode 100644 index 0000000..b9aedd7 --- /dev/null +++ b/tests/jsons/V1_test15.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test16.json b/tests/jsons/V1_test16.json new file mode 100644 index 0000000..9c27d52 --- /dev/null +++ b/tests/jsons/V1_test16.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_file" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test17.json b/tests/jsons/V1_test17.json new file mode 100644 index 0000000..acf861b --- /dev/null +++ b/tests/jsons/V1_test17.json @@ -0,0 +1,23 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode" : ["failMe"] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test18.json b/tests/jsons/V1_test18.json new file mode 100644 index 0000000..e0f3445 --- /dev/null +++ b/tests/jsons/V1_test18.json @@ -0,0 +1,23 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode" : "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test19.json b/tests/jsons/V1_test19.json new file mode 100644 index 0000000..27a3672 --- /dev/null +++ b/tests/jsons/V1_test19.json @@ -0,0 +1,24 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": "failMe" +} \ No newline at end of file diff --git a/tests/jsons/V1_test2.json b/tests/jsons/V1_test2.json new file mode 100644 index 0000000..2e31866 --- /dev/null +++ b/tests/jsons/V1_test2.json @@ -0,0 +1,3 @@ +{ + "name": [] +} \ No newline at end of file diff --git a/tests/jsons/V1_test20.json b/tests/jsons/V1_test20.json new file mode 100644 index 0000000..ed357e9 --- /dev/null +++ b/tests/jsons/V1_test20.json @@ -0,0 +1,26 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1_test21.json b/tests/jsons/V1_test21.json new file mode 100644 index 0000000..fe8a801 --- /dev/null +++ b/tests/jsons/V1_test21.json @@ -0,0 +1,27 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": [], + "fs": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1_test22.json b/tests/jsons/V1_test22.json new file mode 100644 index 0000000..b77b2ba --- /dev/null +++ b/tests/jsons/V1_test22.json @@ -0,0 +1,24 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update", + "n_files" : "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test23.json b/tests/jsons/V1_test23.json new file mode 100644 index 0000000..6f223d4 --- /dev/null +++ b/tests/jsons/V1_test23.json @@ -0,0 +1,23 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "n_files:failMe", + "mode": "no_update" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test3.json b/tests/jsons/V1_test3.json new file mode 100644 index 0000000..71d0d33 --- /dev/null +++ b/tests/jsons/V1_test3.json @@ -0,0 +1,3 @@ +{ + "name" : "test" +} \ No newline at end of file diff --git a/tests/jsons/V1_test4.json b/tests/jsons/V1_test4.json new file mode 100644 index 0000000..90597be --- /dev/null +++ b/tests/jsons/V1_test4.json @@ -0,0 +1,4 @@ +{ + "name" : "test", + "IO_Graph" : "fail" +} \ No newline at end of file diff --git a/tests/jsons/V1_test5.json b/tests/jsons/V1_test5.json new file mode 100644 index 0000000..0230f1a --- /dev/null +++ b/tests/jsons/V1_test5.json @@ -0,0 +1,6 @@ +{ + "name": "test", + "IO_Graph": [ + {} + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test6.json b/tests/jsons/V1_test6.json new file mode 100644 index 0000000..f0c49ea --- /dev/null +++ b/tests/jsons/V1_test6.json @@ -0,0 +1,10 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": [ + "failMe" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test7.json b/tests/jsons/V1_test7.json new file mode 100644 index 0000000..382fe8a --- /dev/null +++ b/tests/jsons/V1_test7.json @@ -0,0 +1,8 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test8.json b/tests/jsons/V1_test8.json new file mode 100644 index 0000000..b647862 --- /dev/null +++ b/tests/jsons/V1_test8.json @@ -0,0 +1,9 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test9.json b/tests/jsons/V1_test9.json new file mode 100644 index 0000000..67cf4f5 --- /dev/null +++ b/tests/jsons/V1_test9.json @@ -0,0 +1,9 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream" : ["file"] + } + ] +} \ No newline at end of file diff --git a/tests/python/test_engine.py b/tests/python/test_engine.py index 48c8375..318283e 100644 --- a/tests/python/test_engine.py +++ b/tests/python/test_engine.py @@ -2,226 +2,226 @@ def test_instantiation(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 + engine = py_capio_cl.Engine() + assert engine.size() == 0 def test_add_file_default(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - - engine.newFile("test.dat") - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + + engine.newFile("test.dat") + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_default_glob(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - - engine.newFile("test.*") - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + + engine.newFile("test.*") + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_default_glob_question(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - - engine.newFile("test.?") - assert engine.size() == 1 - assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.1") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.1") == [] - assert engine.getProducers("test.1") == [] - assert not engine.isPermanent("test.1") - assert not engine.isExcluded("test.1") - assert engine.isFile("test.1") - assert not engine.isDirectory("test.1") - assert engine.getDirectoryFileCount("test.1") == 0 - assert not engine.isStoredInMemory("test.1") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + + engine.newFile("test.?") + assert engine.size() == 1 + assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.1") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.1") == [] + assert engine.getProducers("test.1") == [] + assert not engine.isPermanent("test.1") + assert not engine.isExcluded("test.1") + assert engine.isFile("test.1") + assert not engine.isDirectory("test.1") + assert engine.getDirectoryFileCount("test.1") == 0 + assert not engine.isStoredInMemory("test.1") def test_add_file_manually(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.dat" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, - py_capio_cl.MODE_UPDATE, False, False, file_dependencies) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.dat" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, + py_capio_cl.MODE_UPDATE, False, False, file_dependencies) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_manually_glob(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.*" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, - py_capio_cl.MODE_UPDATE, False, False, file_dependencies) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.*" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, + py_capio_cl.MODE_UPDATE, False, False, file_dependencies) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_manually_question(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.?" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, - py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) - engine.setDirectory("test.?") - engine.setDirectoryFileCount("test.?", 10) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.1") == py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.2") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.2") == py_capio_cl.MODE_NO_UPDATE - assert engine.isDirectory("test.1") - assert engine.getDirectoryFileCount("test.?") == 10 - assert engine.getDirectoryFileCount("test.3") == 10 - assert engine.getConsumers("test.4") == [] - assert engine.getProducers("test.5") == [] - assert not engine.isPermanent("test.6") - assert not engine.isExcluded("test.7") - assert not engine.isFile("test.8") - assert engine.isDirectory("test.9") - assert engine.getDirectoryFileCount("test.a") == 10 - assert not engine.isStoredInMemory("test.b") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.?" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, + py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) + engine.setDirectory("test.?") + engine.setDirectoryFileCount("test.?", 10) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.1") == py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.2") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.2") == py_capio_cl.MODE_NO_UPDATE + assert engine.isDirectory("test.1") + assert engine.getDirectoryFileCount("test.?") == 10 + assert engine.getDirectoryFileCount("test.3") == 10 + assert engine.getConsumers("test.4") == [] + assert engine.getProducers("test.5") == [] + assert not engine.isPermanent("test.6") + assert not engine.isExcluded("test.7") + assert not engine.isFile("test.8") + assert engine.isDirectory("test.9") + assert engine.getDirectoryFileCount("test.a") == 10 + assert not engine.isStoredInMemory("test.b") def test_add_file_manually_glob_explicit(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.[abc][abc][abc]" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, - py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) - engine.setDirectory("test.[abc][abc][abc]") - engine.setDirectoryFileCount("test.[abc][abc][abc]", 10) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.abc") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.cab") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.bac") == py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.ccc") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE - assert engine.isDirectory("test.bbb") - assert engine.getDirectoryFileCount("test.3") != 10 + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.[abc][abc][abc]" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, + py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) + engine.setDirectory("test.[abc][abc][abc]") + engine.setDirectoryFileCount("test.[abc][abc][abc]", 10) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.abc") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.cab") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.bac") == py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.ccc") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE + assert engine.isDirectory("test.bbb") + assert engine.getDirectoryFileCount("test.3") != 10 def test_producers_consumers_file_dependencies(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - producers = ["A", "B"] - consumers = ["C", "D"] - file_dependencies = ["E", "F"] + engine = py_capio_cl.Engine() + assert engine.size() == 0 + producers = ["A", "B"] + consumers = ["C", "D"] + file_dependencies = ["E", "F"] - engine.newFile("test.dat") + engine.newFile("test.dat") - engine.addProducer("test.dat", producers[0]) - assert len(engine.getProducers("test.dat")) == 1 - assert engine.isProducer("test.dat", producers[0]) + engine.addProducer("test.dat", producers[0]) + assert len(engine.getProducers("test.dat")) == 1 + assert engine.isProducer("test.dat", producers[0]) - engine.addProducer("test.dat", producers[1]) - assert len(engine.getProducers("test.dat")) == 2 - assert engine.isProducer("test.dat", producers[1]) + engine.addProducer("test.dat", producers[1]) + assert len(engine.getProducers("test.dat")) == 2 + assert engine.isProducer("test.dat", producers[1]) - engine.addConsumer("test.dat", consumers[0]) - assert len(engine.getConsumers("test.dat")) == 1 - assert engine.isConsumer("test.dat", consumers[0]) + engine.addConsumer("test.dat", consumers[0]) + assert len(engine.getConsumers("test.dat")) == 1 + assert engine.isConsumer("test.dat", consumers[0]) - engine.addConsumer("test.dat", consumers[1]) - assert len(engine.getConsumers("test.dat")) == 2 - assert engine.isConsumer("test.dat", consumers[1]) + engine.addConsumer("test.dat", consumers[1]) + assert len(engine.getConsumers("test.dat")) == 2 + assert engine.isConsumer("test.dat", consumers[1]) - assert engine.getCommitOnFileDependencies("test.dat") == [] - engine.addFileDependency("test.dat", file_dependencies[0]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 - assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] - engine.addFileDependency("test.dat", file_dependencies[1]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 - assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] + assert engine.getCommitOnFileDependencies("test.dat") == [] + engine.addFileDependency("test.dat", file_dependencies[0]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 + assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] + engine.addFileDependency("test.dat", file_dependencies[1]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 + assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] def test_producers_consumers_file_dependencies_glob(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - producers = ["A", "B"] - consumers = ["C", "D"] - file_dependencies = ["E", "F"] - - engine.newFile("test.*") - - engine.addProducer("test.dat", producers[0]) - assert len(engine.getProducers("test.dat")) == 1 - assert engine.isProducer("test.dat", producers[0]) - - engine.addProducer("test.dat", producers[1]) - assert len(engine.getProducers("test.dat")) == 2 - assert engine.isProducer("test.dat", producers[1]) - - engine.addConsumer("test.dat", consumers[0]) - assert len(engine.getConsumers("test.dat")) == 1 - assert engine.isConsumer("test.dat", consumers[0]) - - engine.addConsumer("test.dat", consumers[1]) - assert len(engine.getConsumers("test.dat")) == 2 - assert engine.isConsumer("test.dat", consumers[1]) - - assert engine.getCommitOnFileDependencies("test.dat") == [] - engine.addFileDependency("test.dat", file_dependencies[0]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 - assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] - engine.addFileDependency("test.dat", file_dependencies[1]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 - assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] \ No newline at end of file + engine = py_capio_cl.Engine() + assert engine.size() == 0 + producers = ["A", "B"] + consumers = ["C", "D"] + file_dependencies = ["E", "F"] + + engine.newFile("test.*") + + engine.addProducer("test.dat", producers[0]) + assert len(engine.getProducers("test.dat")) == 1 + assert engine.isProducer("test.dat", producers[0]) + + engine.addProducer("test.dat", producers[1]) + assert len(engine.getProducers("test.dat")) == 2 + assert engine.isProducer("test.dat", producers[1]) + + engine.addConsumer("test.dat", consumers[0]) + assert len(engine.getConsumers("test.dat")) == 1 + assert engine.isConsumer("test.dat", consumers[0]) + + engine.addConsumer("test.dat", consumers[1]) + assert len(engine.getConsumers("test.dat")) == 2 + assert engine.isConsumer("test.dat", consumers[1]) + + assert engine.getCommitOnFileDependencies("test.dat") == [] + engine.addFileDependency("test.dat", file_dependencies[0]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 + assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] + engine.addFileDependency("test.dat", file_dependencies[1]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 + assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1]