From 67090740d8cbdbf5e7be16ca97d114011a401b83 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 13:54:45 +0200 Subject: [PATCH 01/18] Added coverage to project This commits enable the CI action to upload the coverage info to CodeCov and also enables coverage collection with the correct CMake flags. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 013f394..d3de89f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Python Bindings](https://github.com/High-Performance-IO/CAPIO-CL/actions/workflows/python-bindings.yml/badge.svg)](https://github.com/High-Performance-IO/CAPIO-CL/actions/workflows/python-bindings.yml) [![codecov](https://codecov.io/gh/High-Performance-IO/CAPIO-CL/graph/badge.svg)](https://codecov.io/gh/High-Performance-IO/CAPIO-CL) + ![CMake](https://img.shields.io/badge/CMake-%E2%89%A53.15-blue?logo=cmake&logoColor=white) ![C++](https://img.shields.io/badge/C%2B%2B-%E2%89%A517-blueviolet?logo=c%2B%2B&logoColor=white) ![Python Bindings](https://img.shields.io/badge/Python_Bindings-3.10–3.14-darkgreen?style=flat&logo=python&logoColor=white&labelColor=gray) From 4d6887552e56c56464150f047ad404b20ed450ad Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 14:15:09 +0200 Subject: [PATCH 02/18] Enabled coverage in python bindings --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d3de89f..013f394 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ [![Python Bindings](https://github.com/High-Performance-IO/CAPIO-CL/actions/workflows/python-bindings.yml/badge.svg)](https://github.com/High-Performance-IO/CAPIO-CL/actions/workflows/python-bindings.yml) [![codecov](https://codecov.io/gh/High-Performance-IO/CAPIO-CL/graph/badge.svg)](https://codecov.io/gh/High-Performance-IO/CAPIO-CL) - ![CMake](https://img.shields.io/badge/CMake-%E2%89%A53.15-blue?logo=cmake&logoColor=white) ![C++](https://img.shields.io/badge/C%2B%2B-%E2%89%A517-blueviolet?logo=c%2B%2B&logoColor=white) ![Python Bindings](https://img.shields.io/badge/Python_Bindings-3.10–3.14-darkgreen?style=flat&logo=python&logoColor=white&labelColor=gray) From ece6d6137c476571fbbbd4aa89b875775ddcb32c Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 16:46:27 +0200 Subject: [PATCH 03/18] Added new tests --- capiocl.hpp | 2 +- src/Engine.cpp | 6 ++++++ tests/cpp/main.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/capiocl.hpp b/capiocl.hpp index 8459a1a..b65dc97 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -350,7 +350,7 @@ class Engine { bool isConsumer(const std::string &path, const std::string &app_name); /** - * @brief Check if a file is firable. + * @brief Check if a file is firable, that is fire rule is no_update. * * @param path File path. * @return true if the file is firable, false otherwise. diff --git a/src/Engine.cpp b/src/Engine.cpp index fe240d2..d6da6cf 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -269,7 +269,10 @@ void capiocl::Engine::setFireRule(const std::string &path, const std::string &fi START_LOG(gettid(), "call(path=%s, fire_rule=%s)", path.c_str(), fire_rule.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<3>(itm->second) = fire_rule; + return; } + this->newFile(path); + setFireRule(path, fire_rule); } bool capiocl::Engine::isFirable(const std::string &path) { @@ -289,7 +292,10 @@ void capiocl::Engine::setPermanent(const std::string &path, bool value) { START_LOG(gettid(), "call(path=%s, value=%s)", path.c_str(), value ? "true" : "false"); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<4>(itm->second) = value; + return; } + this->newFile(path); + setPermanent(path, value); } bool capiocl::Engine::isPermanent(const std::string &path) { diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index ee571fe..9885e76 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -218,4 +218,42 @@ TEST(testCapioClEngine, testProducerConsumersFileDependenciesGlob) { engine.addFileDependency("test.dat", file_dependencies[1]); EXPECT_EQ(engine.getCommitOnFileDependencies("test.dat").size(), 2); EXPECT_TRUE(engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1]); +} + +TEST(testCapioClEngine, testCommitFirePermanentExcludeOnGlobs) { + capiocl::Engine engine; + engine.newFile("test.*"); + engine.setFireRule("test.*", capiocl::MODE_NO_UPDATE); + + EXPECT_TRUE(engine.isFirable("test.a")); + EXPECT_FALSE(engine.isFirable("testb")); + + engine.setCommitRule("testb", capiocl::COMMITTED_ON_FILE); + engine.setFileDeps("testb", {"test.a"}); + + engine.setPermanent("myFile", true); + EXPECT_TRUE(engine.isPermanent("myFile")); + EXPECT_FALSE(engine.isFirable("myFile")); + + EXPECT_FALSE(engine.isExcluded("testb")); + engine.setExclude("testb", true); + EXPECT_TRUE(engine.isExcluded("testb")); +} + +TEST(testCapioClEngine, testIsFileIsDirectoryGlob) { + capiocl::Engine engine; + engine.newFile("test.*"); + engine.setDirectory("test.d/"); + engine.setDirectory("test.d/bin/lib"); + EXPECT_TRUE(engine.isDirectory("test.d/bin/lib")); + EXPECT_TRUE(engine.isDirectory("test.d/")); + EXPECT_FALSE(engine.isDirectory("test.*")); +} + +TEST(testCapioClEngine, testAddRemoveFile) { + capiocl::Engine engine; + engine.newFile("test.*"); + EXPECT_TRUE(engine.contains("test.*")); + engine.remove("test.*"); + EXPECT_FALSE(engine.contains("test.*")); } \ No newline at end of file From b44969c880f6f2a1177f0d9ae19b3134b6902bd2 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 17:05:06 +0200 Subject: [PATCH 04/18] Added tests and changed unit test pipeline to use docker images with compilers pre installed --- .github/workflows/ci-test.yml | 3 +-- tests/cpp/main.cpp | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index a99495b..3da89c6 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -36,7 +36,6 @@ jobs: check-path: "${{ matrix.path }}" unit-tests: name: "Build ${{ matrix.build_type }} with ${{ matrix.cxx }}" - needs: [ "codespell-check" , "format-check" ] runs-on: ubuntu-22.04 strategy: matrix: @@ -127,7 +126,7 @@ jobs: upload-to-codecov: name: "Codecov report upload" - needs: [ "unit-tests" ] + needs: [ "unit-tests" , "codespell-check" , "format-check" ] runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 9885e76..86c1fdc 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -4,6 +4,7 @@ TEST(testCapioClEngine, testInstantiation) { capiocl::Engine engine; EXPECT_EQ(engine.size(), 0); + engine.print(); } TEST(testCapioClEngine, testAddFileDefault) { @@ -238,6 +239,9 @@ TEST(testCapioClEngine, testCommitFirePermanentExcludeOnGlobs) { EXPECT_FALSE(engine.isExcluded("testb")); engine.setExclude("testb", true); EXPECT_TRUE(engine.isExcluded("testb")); + + engine.setFireRule("test.c", capiocl::MODE_NO_UPDATE); + EXPECT_TRUE(engine.isFirable("test.c")); } TEST(testCapioClEngine, testIsFileIsDirectoryGlob) { From ed8907cba69ce45cd77a253329b464a20a01dd98 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 18:05:33 +0200 Subject: [PATCH 05/18] Added tests --- .github/workflows/ci-test.yml | 5 ++- capiocl.hpp | 2 +- src/Engine.cpp | 59 +++++++++++++++++++++---- tests/cpp/main.cpp | 82 +++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 3da89c6..2664d3e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -111,8 +111,9 @@ jobs: run: | pip install --upgrade gcovr gcovr \ - --exclude-throw-branches \ - --xml coverage.xml \ + --exclude-throw-branches \ + --exclude tests/ \ + --xml coverage.xml \ --gcov-executable "${{ startsWith(matrix.cxx, 'clang-') && format('llvm-cov-{0} gcov', env.CXX_VERSION) || format('gcov-{0}', env.CXX_VERSION) }}" \ ../build - name: "Upload coverage report" diff --git a/capiocl.hpp b/capiocl.hpp index b65dc97..b035091 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -320,7 +320,7 @@ class Engine { std::vector getConsumers(const std::string &path); /// @brief Get the commit-on-close counter for a file. - long getCommitCloseCount(std::filesystem::path::iterator::reference path) const; + long getCommitCloseCount(std::filesystem::path::iterator::reference path); /// @brief Get file dependencies. std::vector getCommitOnFileDependencies(const std::filesystem::path &path); diff --git a/src/Engine.cpp b/src/Engine.cpp index d6da6cf..e86d226 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -203,7 +203,6 @@ long capiocl::Engine::getDirectoryFileCount(const std::string &path) { return std::get<8>(itm->second); } this->newFile(path); - return getDirectoryFileCount(path); } @@ -213,7 +212,10 @@ void capiocl::Engine::addProducer(const std::string &path, std::string &producer newFile(path); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<0>(itm->second).emplace_back(producer); + return; } + this->newFile(path); + this->addProducer(path, producer); } void capiocl::Engine::addConsumer(const std::string &path, std::string &consumer) { @@ -221,22 +223,32 @@ void capiocl::Engine::addConsumer(const std::string &path, std::string &consumer consumer.erase(remove_if(consumer.begin(), consumer.end(), isspace), consumer.end()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<1>(itm->second).emplace_back(consumer); + return; } + this->newFile(path); + this->addConsumer(path, consumer); } + void capiocl::Engine::addFileDependency(const std::string &path, std::string &file_dependency) { START_LOG(gettid(), "call(path=%s, consumer=%s)", path.c_str(), consumer.c_str()); file_dependency.erase(remove_if(file_dependency.begin(), file_dependency.end(), isspace), file_dependency.end()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<9>(itm->second).emplace_back(file_dependency); + return; } + this->newFile(path); + this->addFileDependency(path, file_dependency); } void capiocl::Engine::setCommitRule(const std::string &path, const std::string &commit_rule) { START_LOG(gettid(), "call(path=%s, commit_rule=%s)", path.c_str(), commit_rule.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<2>(itm->second) = commit_rule; + return; } + this->newFile(path); + this->setCommitRule(path, commit_rule); } std::string capiocl::Engine::getCommitRule(const std::string &path) { @@ -319,14 +331,20 @@ void capiocl::Engine::setDirectory(const std::string &path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<6>(itm->second) = false; + return; } + this->newFile(path); + setDirectory(path); } void capiocl::Engine::setFile(const std::string &path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<6>(itm->second) = true; + return; } + this->newFile(path); + setFile(path); } bool capiocl::Engine::isFile(const std::string &path) { @@ -346,18 +364,28 @@ void capiocl::Engine::setCommitedCloseNumber(const std::string &path, const long START_LOG(gettid(), "call(path=%s, num=%ld)", path.c_str(), num); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<7>(itm->second) = num; + return; } + this->newFile(path); + setCommitedCloseNumber(path, num); } void capiocl::Engine::setDirectoryFileCount(const std::string &path, long num) { START_LOG(gettid(), "call(path=%s, num=%ld)", path.c_str(), num); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<8>(itm->second) = num; + return; } + this->newFile(path); + this->setDirectory(path); + this->setDirectoryFileCount(path, num); } void capiocl::Engine::remove(const std::string &path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); + if (const auto itm = _locations.find(path); itm == _locations.end()) { + return; + } _locations.erase(path); } @@ -463,14 +491,17 @@ void capiocl::Engine::setFileDeps(const std::filesystem::path &path, } } -long capiocl::Engine::getCommitCloseCount(std::filesystem::path::iterator::reference path) const { +long capiocl::Engine::getCommitCloseCount(std::filesystem::path::iterator::reference path) { START_LOG(gettid(), "call(path=%s)", path.c_str()); - long count = 0; + if (const auto itm = _locations.find(path); itm != _locations.end()) { - count = std::get<7>(itm->second); + auto count = std::get<7>(itm->second); + LOG("Expected number on close to commit file: %d", count); + return count; } - LOG("Expected number on close to commit file: %d", count); - return count; + + this->newFile(path); + return getCommitCloseCount(path); }; std::vector @@ -482,9 +513,14 @@ capiocl::Engine::getCommitOnFileDependencies(const std::filesystem::path &path) } void capiocl::Engine::setStoreFileInMemory(const std::filesystem::path &path) { + if (const auto itm = _locations.find(path); itm != _locations.end()) { + std::get<10>(_locations.at(path)) = true; + return; + } this->newFile(path); - std::get<10>(_locations.at(path)) = true; + setStoreFileInMemory(path); } + void capiocl::Engine::setAllStoreInMemory() { for (const auto &[fst, snd] : _locations) { this->setStoreFileInMemory(fst); @@ -492,15 +528,20 @@ void capiocl::Engine::setAllStoreInMemory() { } void capiocl::Engine::setStoreFileInFileSystem(const std::filesystem::path &path) { + if (const auto itm = _locations.find(path); itm != _locations.end()) { + std::get<10>(_locations.at(path)) = false; + return; + } this->newFile(path); - std::get<10>(_locations.at(path)) = false; + setStoreFileInFileSystem(path); } bool capiocl::Engine::isStoredInMemory(const std::filesystem::path &path) { if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<10>(itm->second); } - return false; + this->newFile(path); + return isStoredInMemory(path); } std::vector capiocl::Engine::getFileToStoreInMemory() { diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 86c1fdc..2202b80 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -77,6 +77,8 @@ TEST(testCapioClEngine, testAddFileManually) { EXPECT_FALSE(engine.isDirectory("test.dat")); EXPECT_EQ(engine.getDirectoryFileCount("test.dat"), 0); EXPECT_FALSE(engine.isStoredInMemory("test.dat")); + + engine.setFile("test.txt"); } TEST(testCapioClEngine, testAddFileManuallyGlob) { @@ -242,6 +244,8 @@ TEST(testCapioClEngine, testCommitFirePermanentExcludeOnGlobs) { engine.setFireRule("test.c", capiocl::MODE_NO_UPDATE); EXPECT_TRUE(engine.isFirable("test.c")); + + engine.setCommitedCloseNumber("test.e", 100); } TEST(testCapioClEngine, testIsFileIsDirectoryGlob) { @@ -260,4 +264,82 @@ TEST(testCapioClEngine, testAddRemoveFile) { EXPECT_TRUE(engine.contains("test.*")); engine.remove("test.*"); EXPECT_FALSE(engine.contains("test.*")); + engine.remove("data"); + EXPECT_FALSE(engine.contains("data")); +} + +TEST(testCapioClEngine, testProducersConsumers) { + capiocl::Engine engine; + engine.newFile("test.*"); + + std::string consumer = "consumer"; + std::string producer = "producer"; + + engine.addConsumer("test.txt", consumer); + engine.addProducer("test.txt.1", producer); + + EXPECT_TRUE(engine.isProducer("test.txt.1", producer)); + EXPECT_FALSE(engine.isProducer("test.txt.1", consumer)); + + EXPECT_FALSE(engine.isConsumer("test.txt", producer)); + EXPECT_TRUE(engine.isConsumer("test.txt", consumer)); + + engine.addConsumer("test.*", consumer); + engine.addProducer("test.*", producer); + + EXPECT_TRUE(engine.isProducer("test.*", producer)); + EXPECT_FALSE(engine.isProducer("test.*", consumer)); + EXPECT_TRUE(engine.isConsumer("test.*", consumer)); + EXPECT_FALSE(engine.isConsumer("test.*", producer)); +} + +TEST(testCapioClEngine, testCommitCloseCount) { + capiocl::Engine engine; + engine.newFile("test.*"); + engine.setCommitRule("test.*", capiocl::COMMITTED_ON_CLOSE); + engine.setCommitedCloseNumber("test.e", 100); + + EXPECT_EQ(engine.getCommitCloseCount("test.e"), 100); + EXPECT_EQ(engine.getCommitCloseCount("test.d"), 0); + + engine.setCommitedCloseNumber("test.*", 30); + EXPECT_EQ(engine.getCommitCloseCount("test.e"), 30); +} + +TEST(testCapioClEngine, testStorageOptions) { + capiocl::Engine engine; + engine.newFile("A"); + engine.newFile("B"); + + engine.setStoreFileInMemory("A"); + EXPECT_TRUE(engine.isStoredInMemory("A")); + EXPECT_FALSE(engine.isStoredInMemory("B")); + + engine.setStoreFileInMemory("B"); + EXPECT_TRUE(engine.isStoredInMemory("A")); + EXPECT_TRUE(engine.isStoredInMemory("B")); + + engine.setStoreFileInMemory("C"); + EXPECT_TRUE(engine.isStoredInMemory("C")); + + engine.newFile("D"); + EXPECT_FALSE(engine.isStoredInMemory("D")); + + engine.setAllStoreInMemory(); + EXPECT_TRUE(engine.isStoredInMemory("A")); + EXPECT_TRUE(engine.isStoredInMemory("B")); + EXPECT_TRUE(engine.isStoredInMemory("C")); + EXPECT_TRUE(engine.isStoredInMemory("D")); + + EXPECT_EQ(engine.getFileToStoreInMemory().size(), 4); +} + +TEST(testCapioClEngine, testHomeNode) { + std::string nodename; + nodename.reserve(1024); + gethostname(nodename.data(), nodename.size()); + + capiocl::Engine engine; + engine.newFile("A"); + EXPECT_TRUE(engine.getHomeNode("A") == nodename); } \ No newline at end of file From 6d28a0df9cdeb57aa7ec2eb98f176fb3dffb1313 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 18:38:10 +0200 Subject: [PATCH 06/18] Added tests --- tests/cpp/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 2202b80..03b16c9 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -303,7 +303,7 @@ TEST(testCapioClEngine, testCommitCloseCount) { EXPECT_EQ(engine.getCommitCloseCount("test.d"), 0); engine.setCommitedCloseNumber("test.*", 30); - EXPECT_EQ(engine.getCommitCloseCount("test.e"), 30); + EXPECT_EQ(engine.getCommitCloseCount("test.f"), 30); } TEST(testCapioClEngine, testStorageOptions) { From 95232111f9ef9c286d9ce644a154e596ac2f97f5 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 19:22:10 +0200 Subject: [PATCH 07/18] Added tests and bugfixes --- capiocl.hpp | 2 +- src/Engine.cpp | 70 ++++++++++++++-------------------------------- tests/cpp/main.cpp | 35 +++++++++++++++++++++++ 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index b035091..e9a35a2 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -371,7 +371,7 @@ class Engine { * @param path File path. * @return true if excluded, false otherwise. */ - bool isExcluded(const std::string &path) const; + bool isExcluded(const std::string &path); /** * @brief Check if a path is a directory. diff --git a/src/Engine.cpp b/src/Engine.cpp index e86d226..9f976bf 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -209,9 +209,12 @@ long capiocl::Engine::getDirectoryFileCount(const std::string &path) { void capiocl::Engine::addProducer(const std::string &path, std::string &producer) { START_LOG(gettid(), "call(path=%s, producer=%s)", path.c_str(), producer.c_str()); producer.erase(remove_if(producer.begin(), producer.end(), isspace), producer.end()); - newFile(path); + if (const auto itm = _locations.find(path); itm != _locations.end()) { - std::get<0>(itm->second).emplace_back(producer); + if (auto vec = std::get<0>(itm->second); + std::find(vec.begin(), vec.end(), producer) == vec.end()) { + std::get<0>(itm->second).emplace_back(producer); + } return; } this->newFile(path); @@ -222,7 +225,10 @@ void capiocl::Engine::addConsumer(const std::string &path, std::string &consumer START_LOG(gettid(), "call(path=%s, consumer=%s)", path.c_str(), consumer.c_str()); consumer.erase(remove_if(consumer.begin(), consumer.end(), isspace), consumer.end()); if (const auto itm = _locations.find(path); itm != _locations.end()) { - std::get<1>(itm->second).emplace_back(consumer); + if (auto vec = std::get<1>(itm->second); + std::find(vec.begin(), vec.end(), consumer) == vec.end()) { + std::get<1>(itm->second).emplace_back(consumer); + } return; } this->newFile(path); @@ -324,7 +330,10 @@ void capiocl::Engine::setExclude(const std::string &path, const bool value) { START_LOG(gettid(), "call(path=%s, value=%s)", path.c_str(), value ? "true" : "false"); if (const auto itm = _locations.find(path); itm != _locations.end()) { std::get<5>(itm->second) = value; + return; } + this->newFile(path); + setExclude(path, value); } void capiocl::Engine::setDirectory(const std::string &path) { @@ -415,21 +424,8 @@ bool capiocl::Engine::isConsumer(const std::string &path, const std::string &app } LOG("No exact match found in locations. checking for globs"); - // check for glob. Here we do not use the LMP check - for (const auto &[k, entry] : _locations) { - if (fnmatch(k.c_str(), path.c_str(), FNM_PATHNAME) == 0) { - LOG("Found possible glob match"); - std::vector producers = std::get<1>(entry); - DBG(gettid(), [&](const std::vector &arr) { - for (auto itm : arr) { - LOG("producer: %s", itm.c_str()); - } - }(producers)); - return std::find(producers.begin(), producers.end(), app_name) != producers.end(); - } - } - LOG("No match has been found"); - return false; + this->newFile(path); + return isConsumer(path, app_name); } std::vector capiocl::Engine::getProducers(const std::string &path) { @@ -437,7 +433,8 @@ std::vector capiocl::Engine::getProducers(const std::string &path) if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<0>(itm->second); } - return {}; + this->newFile(path); + return getProducers(path); } bool capiocl::Engine::isProducer(const std::string &path, const std::string &app_name) { @@ -458,21 +455,8 @@ bool capiocl::Engine::isProducer(const std::string &path, const std::string &app } LOG("No exact match found in locations. checking for globs"); - // check for glob. Here we do not use the LMP check - for (const auto &[k, entry] : _locations) { - if (fnmatch(k.c_str(), path.c_str(), FNM_PATHNAME) == 0) { - LOG("Found possible glob match"); - std::vector producers = std::get<0>(entry); - DBG(gettid(), [&](const std::vector &arr) { - for (auto itm : arr) { - LOG("producer: %s", itm.c_str()); - } - }(producers)); - return std::find(producers.begin(), producers.end(), app_name) != producers.end(); - } - } - LOG("No match has been found"); - return false; + this->newFile(path); + return isProducer(path, app_name); } void capiocl::Engine::setFileDeps(const std::filesystem::path &path, @@ -570,25 +554,13 @@ std::string capiocl::Engine::getHomeNode(const std::string &path) { return node_name; } -bool capiocl::Engine::isExcluded(const std::string &path) const { +bool capiocl::Engine::isExcluded(const std::string &path) { if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<5>(itm->second); } LOG("Checking against GLOB"); - size_t lpm_match_size = -1; - bool lpm_match = false; - for (const auto &[glob_path, entry] : _locations) { - if (fnmatch(glob_path.c_str(), path.c_str(), FNM_PATHNAME) == 0) { - LOG("Found match with %s", glob_path.c_str()); - if (glob_path.length() > lpm_match_size) { - lpm_match_size = glob_path.length(); - lpm_match = std::get<5>(entry); - LOG("Match is longer than previous match. storing value = %s", - lpm_match ? "true" : "false"); - } - } - } - return lpm_match; + this->newFile(path); + return isExcluded(path); } diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 03b16c9..e4d61d0 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -131,6 +131,9 @@ TEST(testCapioClEngine, testAddFileManuallyQuestion) { EXPECT_TRUE(engine.isDirectory("test.9")); EXPECT_EQ(engine.getDirectoryFileCount("test.a"), 10); EXPECT_FALSE(engine.isStoredInMemory("test.b")); + + engine.setDirectoryFileCount("myDir", 10); + EXPECT_EQ(engine.getDirectoryFileCount("myDir"), 10); } TEST(testCapioClEngine, testAddFileManuallyGlobExplcit) { @@ -188,6 +191,8 @@ TEST(testCapioClEngine, testProducerConsumersFileDependencies) { engine.addFileDependency("test.dat", file_dependencies[1]); EXPECT_EQ(engine.getCommitOnFileDependencies("test.dat").size(), 2); EXPECT_TRUE(engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1]); + + EXPECT_TRUE(engine.getCommitOnFileDependencies("myNewFile").empty()); } TEST(testCapioClEngine, testProducerConsumersFileDependenciesGlob) { @@ -241,6 +246,11 @@ TEST(testCapioClEngine, testCommitFirePermanentExcludeOnGlobs) { EXPECT_FALSE(engine.isExcluded("testb")); engine.setExclude("testb", true); EXPECT_TRUE(engine.isExcluded("testb")); + engine.setExclude("testb", true); + EXPECT_TRUE(engine.isExcluded("testb")); + engine.setExclude("myFile.*", true); + EXPECT_TRUE(engine.isExcluded("myFile.txt")); + EXPECT_TRUE(engine.isExcluded("myFile.dat")); engine.setFireRule("test.c", capiocl::MODE_NO_UPDATE); EXPECT_TRUE(engine.isFirable("test.c")); @@ -291,6 +301,19 @@ TEST(testCapioClEngine, testProducersConsumers) { EXPECT_FALSE(engine.isProducer("test.*", consumer)); EXPECT_TRUE(engine.isConsumer("test.*", consumer)); EXPECT_FALSE(engine.isConsumer("test.*", producer)); + + engine.addProducer("test.k", producer); + engine.addConsumer("test.k", consumer); + EXPECT_TRUE(engine.isProducer("test.k", producer)); + EXPECT_TRUE(engine.isConsumer("test.k", consumer)); + + EXPECT_TRUE(engine.isConsumer("test.txt.2", consumer)); + EXPECT_FALSE(engine.isProducer("test.txt.3", consumer)); + EXPECT_FALSE(engine.isConsumer("test.txt.4", producer)); + EXPECT_TRUE(engine.isProducer("test.txt.4", producer)); + + EXPECT_EQ(engine.getProducers("myNewFile").size(), 0); + EXPECT_EQ(engine.getProducers("test.k").size(), 1); } TEST(testCapioClEngine, testCommitCloseCount) { @@ -304,6 +327,9 @@ TEST(testCapioClEngine, testCommitCloseCount) { engine.setCommitedCloseNumber("test.*", 30); EXPECT_EQ(engine.getCommitCloseCount("test.f"), 30); + + engine.setCommitRule("myFile", capiocl::COMMITTED_ON_FILE); + EXPECT_TRUE(engine.getCommitRule("myFile") == capiocl::COMMITTED_ON_FILE); } TEST(testCapioClEngine, testStorageOptions) { @@ -332,6 +358,15 @@ TEST(testCapioClEngine, testStorageOptions) { EXPECT_TRUE(engine.isStoredInMemory("D")); EXPECT_EQ(engine.getFileToStoreInMemory().size(), 4); + + engine.setStoreFileInFileSystem("A"); + engine.setStoreFileInFileSystem("B"); + engine.setStoreFileInFileSystem("C"); + engine.setStoreFileInFileSystem("D"); + EXPECT_FALSE(engine.isStoredInMemory("A")); + EXPECT_FALSE(engine.isStoredInMemory("B")); + EXPECT_FALSE(engine.isStoredInMemory("C")); + EXPECT_FALSE(engine.isStoredInMemory("D")); } TEST(testCapioClEngine, testHomeNode) { From c26278fab4685d2ad25ce3058c702a2d184f4a0b Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 19:46:35 +0200 Subject: [PATCH 08/18] Added tests and bugfixes --- src/Engine.cpp | 15 ++++++++++----- tests/cpp/main.cpp | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/Engine.cpp b/src/Engine.cpp index 9f976bf..5280c2b 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -244,6 +244,7 @@ void capiocl::Engine::addFileDependency(const std::string &path, std::string &fi return; } this->newFile(path); + this->setCommitRule(path, capiocl::COMMITTED_ON_FILE); this->addFileDependency(path, file_dependency); } @@ -465,14 +466,18 @@ void capiocl::Engine::setFileDeps(const std::filesystem::path &path, if (dependencies.empty()) { return; } - if (_locations.find(path) == _locations.end()) { - this->newFile(path); - } - std::get<9>(_locations.at(path)) = dependencies; + for (const auto &itm : dependencies) { - LOG("Creating new fie (if it exists) for path %s", itm.c_str()); + LOG("Creating new file for path %s", itm.c_str()); newFile(itm); } + + if (_locations.find(path) != _locations.end()) { + std::get<9>(_locations.at(path)) = dependencies; + return; + } + this->newFile(path); + setFileDeps(path, dependencies); } long capiocl::Engine::getCommitCloseCount(std::filesystem::path::iterator::reference path) { diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index e4d61d0..2fa171d 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -193,6 +193,11 @@ TEST(testCapioClEngine, testProducerConsumersFileDependencies) { EXPECT_TRUE(engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1]); EXPECT_TRUE(engine.getCommitOnFileDependencies("myNewFile").empty()); + + engine.addFileDependency("myFile.txt", file_dependencies[0]); + EXPECT_TRUE(engine.getCommitRule("myFile.txt") == capiocl::COMMITTED_ON_FILE); + EXPECT_EQ(engine.getCommitOnFileDependencies("myFile.txt").size(), 1); + EXPECT_TRUE(engine.getCommitOnFileDependencies("myFile.txt")[0] == file_dependencies[0]); } TEST(testCapioClEngine, testProducerConsumersFileDependenciesGlob) { @@ -377,4 +382,15 @@ TEST(testCapioClEngine, testHomeNode) { capiocl::Engine engine; engine.newFile("A"); EXPECT_TRUE(engine.getHomeNode("A") == nodename); +} + +TEST(testCapioClEngine, testInsertFileDependencies) { + capiocl::Engine engine; + + engine.setFileDeps("myFile.txt", {}); + engine.setFileDeps("test.txt", {"a", "b", "c"}); + EXPECT_EQ(engine.getCommitOnFileDependencies("test.txt").size(), 3); + EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[0] == "a"); + EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[1] == "b"); + EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[2] == "c"); } \ No newline at end of file From 042b65468018684a4deec1d18266eb9bbfed29b4 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 8 Oct 2025 20:34:13 +0200 Subject: [PATCH 09/18] Began adding paser and serializer test. now broken... --- capiocl.hpp | 19 +++++++++++++++++++ src/Engine.cpp | 2 +- tests/cpp/main.cpp | 47 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index e9a35a2..9546700 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -396,6 +396,25 @@ class Engine { * @return */ bool isPermanent(const std::string &path); + + bool operator==(const capiocl::Engine &other) const { + const auto &other_locations = other.getLocations(); + + // check same size + if (this->_locations.size() != other_locations->size()) { + return false; + } + + // check same entry paths + for (const auto &[this_path, this_itm] : this->_locations) { + if (other_locations->find(this_path) == other_locations->end()) { + return false; + } + // check same config for each path + auto other_itm = other_locations->at(this_path); + } + return true; + } }; /** diff --git a/src/Engine.cpp b/src/Engine.cpp index 5280c2b..0dca1da 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -128,7 +128,7 @@ void capiocl::Engine::print() const { bool capiocl::Engine::contains(const std::filesystem::path &file) { START_LOG(gettid(), "call(file=%s)", file.c_str()); for (auto &[fst, snd] : _locations) { - if (fnmatch(file.c_str(), fst.c_str(), FNM_PATHNAME) == 0) { + if (fnmatch(fst.c_str(), file.c_str(), FNM_PATHNAME) == 0) { return true; } } diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 2fa171d..a479ccf 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -277,6 +277,7 @@ TEST(testCapioClEngine, testAddRemoveFile) { capiocl::Engine engine; engine.newFile("test.*"); EXPECT_TRUE(engine.contains("test.*")); + EXPECT_TRUE(engine.contains("test.txt")); engine.remove("test.*"); EXPECT_FALSE(engine.contains("test.*")); engine.remove("data"); @@ -372,6 +373,9 @@ TEST(testCapioClEngine, testStorageOptions) { EXPECT_FALSE(engine.isStoredInMemory("B")); EXPECT_FALSE(engine.isStoredInMemory("C")); EXPECT_FALSE(engine.isStoredInMemory("D")); + + engine.setStoreFileInFileSystem("F"); + EXPECT_FALSE(engine.isStoredInMemory("F")); } TEST(testCapioClEngine, testHomeNode) { @@ -393,4 +397,45 @@ TEST(testCapioClEngine, testInsertFileDependencies) { EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[0] == "a"); EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[1] == "b"); EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[2] == "c"); -} \ No newline at end of file +} + +/* +TEST(testCapioSerializerParser, testSerializeParse) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "my_command_history.txt"; + std::string producer_name = "_first", consumer_name = "_last", intermediate_name = "_middle"; + + capiocl::Engine engine; + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, intermediate_name); + engine.addProducer(file_2_name, intermediate_name); + engine.addConsumer(file_2_name, consumer_name); + engine.addConsumer(file_1_name, consumer_name); + + engine.setStoreFileInMemory(file_1_name); + engine.setCommitRule(file_1_name, capiocl::COMMITTED_ON_CLOSE); + engine.setCommitedCloseNumber(file_1_name, 3); + engine.setFireRule(file_1_name, capiocl::MODE_UPDATE); + engine.setPermanent(file_1_name, true); + + engine.setStoreFileInFileSystem(file_2_name); + engine.setCommitRule(file_2_name, capiocl::COMMITTED_ON_TERMINATION); + engine.setFireRule(file_1_name, capiocl::MODE_NO_UPDATE); + + engine.addProducer(file_3_name, producer_name); + engine.addProducer(file_3_name, consumer_name); + engine.addProducer(file_3_name, intermediate_name); + engine.setExclude(file_3_name, true); + engine.print(); + + capiocl::Serializer::dump(engine, workflow_name, path); + + std::filesystem::path resolve = ""; + auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); + + EXPECT_TRUE(wf_name == workflow_name); + EXPECT_TRUE(engine == *new_engine); +} +*/ \ No newline at end of file From 5f2929097d9c2aa8bb48189cf0d5f83fbe308263 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 10:00:06 +0200 Subject: [PATCH 10/18] TestSerializeParse --- capiocl.hpp | 19 +- src/Engine.cpp | 91 +++++++++ src/Parser.cpp | 25 +-- src/Serializer.cpp | 11 -- tests/cpp/main.cpp | 8 +- tests/python/test_engine.py | 380 ++++++++++++++++++------------------ 6 files changed, 294 insertions(+), 240 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index 9546700..c946082 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -397,24 +397,7 @@ class Engine { */ bool isPermanent(const std::string &path); - bool operator==(const capiocl::Engine &other) const { - const auto &other_locations = other.getLocations(); - - // check same size - if (this->_locations.size() != other_locations->size()) { - return false; - } - - // check same entry paths - for (const auto &[this_path, this_itm] : this->_locations) { - if (other_locations->find(this_path) == other_locations->end()) { - return false; - } - // check same config for each path - auto other_itm = other_locations->at(this_path); - } - return true; - } + bool operator==(const capiocl::Engine &other) const; }; /** diff --git a/src/Engine.cpp b/src/Engine.cpp index 0dca1da..a342061 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -569,3 +569,94 @@ bool capiocl::Engine::isExcluded(const std::string &path) { this->newFile(path); return isExcluded(path); } + +bool capiocl::Engine::operator==(const capiocl::Engine &other) const { + const auto &other_locations = other.getLocations(); + + // check same size + if (this->_locations.size() != other_locations->size()) { + return false; + } + + // check same entry paths + for (const auto &[this_path, this_itm] : this->_locations) { + if (other_locations->find(this_path) == other_locations->end()) { + return false; + } + // check same config for each path + auto other_itm = other_locations->at(this_path); + + // check for "primitive" data types + if (std::get<2>(this_itm) != std::get<2>(other_itm)) { + return false; + } + + if (std::get<3>(this_itm) != std::get<3>(other_itm)) { + return false; + } + + if (std::get<4>(this_itm) != std::get<4>(other_itm)) { + return false; + } + + if (std::get<5>(this_itm) != std::get<5>(other_itm)) { + return false; + } + + if (std::get<6>(this_itm) != std::get<6>(other_itm)) { + return false; + } + + if (std::get<7>(this_itm) != std::get<7>(other_itm)) { + return false; + } + + if (std::get<8>(this_itm) != std::get<8>(other_itm)) { + return false; + } + + if (std::get<10>(this_itm) != std::get<10>(other_itm)) { + return false; + } + + // check for producer vector + auto this_producer = std::get<0>(this_itm); + auto other_producer = std::get<0>(this_itm); + if (this_producer.size() != other_producer.size()) { + return false; + } + for (const auto &entry : this_producer) { + if (std::find(other_producer.begin(), other_producer.end(), entry) == + other_producer.end()) { + return false; + } + } + + // check for consumer vector + auto this_consumer = std::get<1>(this_itm); + auto other_consumer = std::get<1>(this_itm); + if (this_consumer.size() != other_consumer.size()) { + return false; + } + for (const auto &entry : this_consumer) { + if (std::find(other_consumer.begin(), other_consumer.end(), entry) == + other_consumer.end()) { + return false; + } + } + + // check for file dependencies + + auto this_deps = std::get<9>(this_itm); + auto other_deps = std::get<9>(this_itm); + if (this_deps.size() != other_deps.size()) { + return false; + } + for (const auto &entry : this_deps) { + if (std::find(other_deps.begin(), other_deps.end(), entry) == other_deps.end()) { + return false; + } + } + } + return true; +} \ No newline at end of file diff --git a/src/Parser.cpp b/src/Parser.cpp index 76daaf4..b5e6193 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -24,20 +24,11 @@ std::tuple capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path &resolve_prefix, bool store_only_in_memory) { + bool skip_resolve = resolve_prefix.empty(); std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto locations = new Engine(); START_LOG(gettid(), "call(config_file='%s')", source.c_str()); - if (resolve_prefix.empty()) { - resolve_prefix = "."; - } - - locations->newFile("*"); - locations->setDirectory("*"); - if (store_only_in_memory) { - locations->setStoreFileInMemory("*"); - } - if (source.empty()) { return {workflow_name, locations}; } @@ -95,7 +86,7 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat print_message(CLI_LEVEL_JSON, "Parsing input_stream for app " + app_name); for (const auto &itm : app["input_stream"]) { std::filesystem::path file_path(itm.get()); - if (file_path.is_relative()) { + if (file_path.is_relative() && !skip_resolve) { print_message(CLI_LEVEL_WARNING, "Path : " + file_path.string() + " IS RELATIVE! resolving..."); file_path = resolve_prefix / file_path; @@ -113,7 +104,7 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat print_message(CLI_LEVEL_JSON, "Parsing output_stream for app " + app_name); for (const auto &itm : app["output_stream"]) { std::filesystem::path file_path(itm.get()); - if (file_path.is_relative()) { + if (file_path.is_relative() && !skip_resolve) { print_message(CLI_LEVEL_WARNING, "Path : " + file_path.string() + " IS RELATIVE! resolving..."); file_path = resolve_prefix / file_path; @@ -130,14 +121,14 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat std::vector streaming_names; std::vector file_deps; std::string commit_rule = COMMITTED_ON_TERMINATION, mode = MODE_UPDATE; - long int n_close = -1; - int64_t n_files = -1; + long int n_close = 0; + int64_t n_files = 0; // name or dirname if (stream_item.contains("name") && stream_item["name"].is_array()) { for (const auto &nm : stream_item["name"]) { std::filesystem::path p(nm.get()); - if (p.is_relative()) { + if (p.is_relative() && !skip_resolve) { p = resolve_prefix / p; } streaming_names.push_back(p); @@ -146,7 +137,7 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat is_file = false; for (const auto &nm : stream_item["dirname"]) { std::filesystem::path p(nm.get()); - if (p.is_relative()) { + if (p.is_relative() && !skip_resolve) { p = resolve_prefix / p; } streaming_names.push_back(p); @@ -193,7 +184,7 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat } for (const auto &dep : stream_item["file_deps"]) { std::filesystem::path p(dep.get()); - if (p.is_relative()) { + if (p.is_relative() && !skip_resolve) { p = resolve_prefix / p; } file_deps.push_back(p); diff --git a/src/Serializer.cpp b/src/Serializer.cpp index 89d989b..bc977ea 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -29,11 +29,6 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string for (const auto &[path, data] : *files) { const auto &[producers, consumers, commit_rule, fire_rule, permanent_flag, excluded_flag, is_file, n_close, n_dir_files, file_deps, store_in_memory] = data; - if (path == "*") { - LOG("Skipping * path"); - print_message(CLI_LEVEL_WARNING, "Skipping * path"); - continue; - } // Collect permanent/exclude info if (permanent_flag) { @@ -81,12 +76,6 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string excluded_flag, is_file, n_close, n_dir_files, file_deps, store_in_fs] = data; - if (path == "*") { - LOG("Skipping * path"); - print_message(CLI_LEVEL_WARNING, "Skipping * path"); - continue; - } - nlohmann::json sitem; if (is_file) { sitem["name"] = nlohmann::json::array({path}); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index a479ccf..48b3d78 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -399,8 +399,7 @@ TEST(testCapioClEngine, testInsertFileDependencies) { EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[2] == "c"); } -/* -TEST(testCapioSerializerParser, testSerializeParse) { +TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { const std::filesystem::path path("./config.json"); const std::string workflow_name = "demo"; const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", @@ -433,9 +432,10 @@ TEST(testCapioSerializerParser, testSerializeParse) { capiocl::Serializer::dump(engine, workflow_name, path); std::filesystem::path resolve = ""; - auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); + auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); EXPECT_TRUE(wf_name == workflow_name); EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); } -*/ \ No newline at end of file diff --git a/tests/python/test_engine.py b/tests/python/test_engine.py index 48c8375..318283e 100644 --- a/tests/python/test_engine.py +++ b/tests/python/test_engine.py @@ -2,226 +2,226 @@ def test_instantiation(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 + engine = py_capio_cl.Engine() + assert engine.size() == 0 def test_add_file_default(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - - engine.newFile("test.dat") - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + + engine.newFile("test.dat") + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_default_glob(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - - engine.newFile("test.*") - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + + engine.newFile("test.*") + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_default_glob_question(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - - engine.newFile("test.?") - assert engine.size() == 1 - assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.1") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.1") == [] - assert engine.getProducers("test.1") == [] - assert not engine.isPermanent("test.1") - assert not engine.isExcluded("test.1") - assert engine.isFile("test.1") - assert not engine.isDirectory("test.1") - assert engine.getDirectoryFileCount("test.1") == 0 - assert not engine.isStoredInMemory("test.1") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + + engine.newFile("test.?") + assert engine.size() == 1 + assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.1") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.1") == [] + assert engine.getProducers("test.1") == [] + assert not engine.isPermanent("test.1") + assert not engine.isExcluded("test.1") + assert engine.isFile("test.1") + assert not engine.isDirectory("test.1") + assert engine.getDirectoryFileCount("test.1") == 0 + assert not engine.isStoredInMemory("test.1") def test_add_file_manually(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.dat" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, - py_capio_cl.MODE_UPDATE, False, False, file_dependencies) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.dat" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, + py_capio_cl.MODE_UPDATE, False, False, file_dependencies) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_manually_glob(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.*" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, - py_capio_cl.MODE_UPDATE, False, False, file_dependencies) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION - assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE - assert engine.getConsumers("test.dat") == [] - assert engine.getProducers("test.dat") == [] - assert not engine.isPermanent("test.dat") - assert not engine.isExcluded("test.dat") - assert engine.isFile("test.dat") - assert not engine.isDirectory("test.dat") - assert engine.getDirectoryFileCount("test.dat") == 0 - assert not engine.isStoredInMemory("test.dat") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.*" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_TERMINATION, + py_capio_cl.MODE_UPDATE, False, False, file_dependencies) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") == py_capio_cl.COMMITTED_ON_TERMINATION + assert engine.getFireRule("test.dat") == py_capio_cl.MODE_UPDATE + assert engine.getConsumers("test.dat") == [] + assert engine.getProducers("test.dat") == [] + assert not engine.isPermanent("test.dat") + assert not engine.isExcluded("test.dat") + assert engine.isFile("test.dat") + assert not engine.isDirectory("test.dat") + assert engine.getDirectoryFileCount("test.dat") == 0 + assert not engine.isStoredInMemory("test.dat") def test_add_file_manually_question(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.?" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, - py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) - engine.setDirectory("test.?") - engine.setDirectoryFileCount("test.?", 10) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.1") == py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.2") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.2") == py_capio_cl.MODE_NO_UPDATE - assert engine.isDirectory("test.1") - assert engine.getDirectoryFileCount("test.?") == 10 - assert engine.getDirectoryFileCount("test.3") == 10 - assert engine.getConsumers("test.4") == [] - assert engine.getProducers("test.5") == [] - assert not engine.isPermanent("test.6") - assert not engine.isExcluded("test.7") - assert not engine.isFile("test.8") - assert engine.isDirectory("test.9") - assert engine.getDirectoryFileCount("test.a") == 10 - assert not engine.isStoredInMemory("test.b") + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.?" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, + py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) + engine.setDirectory("test.?") + engine.setDirectoryFileCount("test.?", 10) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.1") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.1") == py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.2") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.2") == py_capio_cl.MODE_NO_UPDATE + assert engine.isDirectory("test.1") + assert engine.getDirectoryFileCount("test.?") == 10 + assert engine.getDirectoryFileCount("test.3") == 10 + assert engine.getConsumers("test.4") == [] + assert engine.getProducers("test.5") == [] + assert not engine.isPermanent("test.6") + assert not engine.isExcluded("test.7") + assert not engine.isFile("test.8") + assert engine.isDirectory("test.9") + assert engine.getDirectoryFileCount("test.a") == 10 + assert not engine.isStoredInMemory("test.b") def test_add_file_manually_glob_explicit(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - path = "test.[abc][abc][abc]" - producers, consumers, file_dependencies = [], [], [] - - engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, - py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) - engine.setDirectory("test.[abc][abc][abc]") - engine.setDirectoryFileCount("test.[abc][abc][abc]", 10) - - assert engine.size() == 1 - assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.abc") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.cab") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.bac") == py_capio_cl.MODE_NO_UPDATE - assert engine.getCommitRule("test.ccc") == py_capio_cl.COMMITTED_ON_CLOSE - assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE - assert engine.isDirectory("test.bbb") - assert engine.getDirectoryFileCount("test.3") != 10 + engine = py_capio_cl.Engine() + assert engine.size() == 0 + path = "test.[abc][abc][abc]" + producers, consumers, file_dependencies = [], [], [] + + engine.add(path, producers, consumers, py_capio_cl.COMMITTED_ON_CLOSE, + py_capio_cl.MODE_NO_UPDATE, False, False, file_dependencies) + engine.setDirectory("test.[abc][abc][abc]") + engine.setDirectoryFileCount("test.[abc][abc][abc]", 10) + + assert engine.size() == 1 + assert engine.getCommitRule("test.dat") != py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.dat") != py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.abc") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.cab") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.bac") == py_capio_cl.MODE_NO_UPDATE + assert engine.getCommitRule("test.ccc") == py_capio_cl.COMMITTED_ON_CLOSE + assert engine.getFireRule("test.aaa") == py_capio_cl.MODE_NO_UPDATE + assert engine.isDirectory("test.bbb") + assert engine.getDirectoryFileCount("test.3") != 10 def test_producers_consumers_file_dependencies(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - producers = ["A", "B"] - consumers = ["C", "D"] - file_dependencies = ["E", "F"] + engine = py_capio_cl.Engine() + assert engine.size() == 0 + producers = ["A", "B"] + consumers = ["C", "D"] + file_dependencies = ["E", "F"] - engine.newFile("test.dat") + engine.newFile("test.dat") - engine.addProducer("test.dat", producers[0]) - assert len(engine.getProducers("test.dat")) == 1 - assert engine.isProducer("test.dat", producers[0]) + engine.addProducer("test.dat", producers[0]) + assert len(engine.getProducers("test.dat")) == 1 + assert engine.isProducer("test.dat", producers[0]) - engine.addProducer("test.dat", producers[1]) - assert len(engine.getProducers("test.dat")) == 2 - assert engine.isProducer("test.dat", producers[1]) + engine.addProducer("test.dat", producers[1]) + assert len(engine.getProducers("test.dat")) == 2 + assert engine.isProducer("test.dat", producers[1]) - engine.addConsumer("test.dat", consumers[0]) - assert len(engine.getConsumers("test.dat")) == 1 - assert engine.isConsumer("test.dat", consumers[0]) + engine.addConsumer("test.dat", consumers[0]) + assert len(engine.getConsumers("test.dat")) == 1 + assert engine.isConsumer("test.dat", consumers[0]) - engine.addConsumer("test.dat", consumers[1]) - assert len(engine.getConsumers("test.dat")) == 2 - assert engine.isConsumer("test.dat", consumers[1]) + engine.addConsumer("test.dat", consumers[1]) + assert len(engine.getConsumers("test.dat")) == 2 + assert engine.isConsumer("test.dat", consumers[1]) - assert engine.getCommitOnFileDependencies("test.dat") == [] - engine.addFileDependency("test.dat", file_dependencies[0]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 - assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] - engine.addFileDependency("test.dat", file_dependencies[1]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 - assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] + assert engine.getCommitOnFileDependencies("test.dat") == [] + engine.addFileDependency("test.dat", file_dependencies[0]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 + assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] + engine.addFileDependency("test.dat", file_dependencies[1]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 + assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] def test_producers_consumers_file_dependencies_glob(): - engine = py_capio_cl.Engine() - assert engine.size() == 0 - producers = ["A", "B"] - consumers = ["C", "D"] - file_dependencies = ["E", "F"] - - engine.newFile("test.*") - - engine.addProducer("test.dat", producers[0]) - assert len(engine.getProducers("test.dat")) == 1 - assert engine.isProducer("test.dat", producers[0]) - - engine.addProducer("test.dat", producers[1]) - assert len(engine.getProducers("test.dat")) == 2 - assert engine.isProducer("test.dat", producers[1]) - - engine.addConsumer("test.dat", consumers[0]) - assert len(engine.getConsumers("test.dat")) == 1 - assert engine.isConsumer("test.dat", consumers[0]) - - engine.addConsumer("test.dat", consumers[1]) - assert len(engine.getConsumers("test.dat")) == 2 - assert engine.isConsumer("test.dat", consumers[1]) - - assert engine.getCommitOnFileDependencies("test.dat") == [] - engine.addFileDependency("test.dat", file_dependencies[0]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 - assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] - engine.addFileDependency("test.dat", file_dependencies[1]) - assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 - assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] \ No newline at end of file + engine = py_capio_cl.Engine() + assert engine.size() == 0 + producers = ["A", "B"] + consumers = ["C", "D"] + file_dependencies = ["E", "F"] + + engine.newFile("test.*") + + engine.addProducer("test.dat", producers[0]) + assert len(engine.getProducers("test.dat")) == 1 + assert engine.isProducer("test.dat", producers[0]) + + engine.addProducer("test.dat", producers[1]) + assert len(engine.getProducers("test.dat")) == 2 + assert engine.isProducer("test.dat", producers[1]) + + engine.addConsumer("test.dat", consumers[0]) + assert len(engine.getConsumers("test.dat")) == 1 + assert engine.isConsumer("test.dat", consumers[0]) + + engine.addConsumer("test.dat", consumers[1]) + assert len(engine.getConsumers("test.dat")) == 2 + assert engine.isConsumer("test.dat", consumers[1]) + + assert engine.getCommitOnFileDependencies("test.dat") == [] + engine.addFileDependency("test.dat", file_dependencies[0]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 1 + assert engine.getCommitOnFileDependencies("test.dat")[0] == file_dependencies[0] + engine.addFileDependency("test.dat", file_dependencies[1]) + assert len(engine.getCommitOnFileDependencies("test.dat")) == 2 + assert engine.getCommitOnFileDependencies("test.dat")[1] == file_dependencies[1] From 28c99e9fc3803d79cf20af610367e1ea8169e091 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 10:32:33 +0200 Subject: [PATCH 11/18] Tests on == operator --- src/Engine.cpp | 31 +++++++++++---------------- tests/cpp/main.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/src/Engine.cpp b/src/Engine.cpp index a342061..97b66fb 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -10,9 +10,7 @@ void capiocl::Engine::print() const { print_message(CLI_LEVEL_JSON, "Composition of expected CAPIO FS: "); // Table header lines - print_message(CLI_LEVEL_JSON, - "|============================================================================" - "==========================================================|"); + print_message(CLI_LEVEL_JSON, "*" + std::string(134, '=') + "*"); print_message(CLI_LEVEL_JSON, "|" + std::string(134, ' ') + "|"); @@ -34,9 +32,7 @@ void capiocl::Engine::print() const { "| \033[48;5;172m \033[0m File stored on file system" + std::string(77, ' ') + "|"); - print_message(CLI_LEVEL_JSON, - "|============================================================================" - "==========================================================|"); + print_message(CLI_LEVEL_JSON, "|" + std::string(134, '=') + "|"); print_message(CLI_LEVEL_JSON, "|======|===================|===================|====================|========" @@ -101,25 +97,22 @@ void capiocl::Engine::print() const { fire_rule = std::get<3>(itm.second); bool exclude = std::get<5>(itm.second), permanent = std::get<4>(itm.second); - line << " " << commit_rule << std::setfill(' ') - << std::setw(20 - commit_rule.length()) << " | " << fire_rule - << std::setfill(' ') << std::setw(13 - fire_rule.length()) << " | " - << " " << (permanent ? "YES" : "NO ") << " | " - << (exclude ? "YES" : "NO ") << " | " << n_files - << std::setw(10 - n_files.length()) << " |"; + line << " " << commit_rule << std::setfill(' '); + line << std::setw(20 - commit_rule.length()) << " | " << fire_rule; + line << std::setfill(' ') << std::setw(13 - fire_rule.length()) << " | "; + line << " " << (permanent ? "YES" : "NO ") << " | "; + line << (exclude ? "YES" : "NO "); + line << " | " << n_files << std::setw(10 - n_files.length()) << " |"; } else { - line << std::setfill(' ') << std::setw(20) << "|" << std::setfill(' ') - << std::setw(13) << "|" << std::setfill(' ') << std::setw(12) << "|" - << std::setfill(' ') << std::setw(10) << "|" << std::setw(11) << "|"; + line << std::setfill(' ') << std::setw(20) << "|" << std::setfill(' '); + line << std::setw(13) << "|" << std::setfill(' ') << std::setw(12) << "|"; + line << std::setfill(' ') << std::setw(10) << "|" << std::setw(11) << "|"; } print_message(CLI_LEVEL_JSON, line.str()); } - print_message(CLI_LEVEL_JSON, - "*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - "~~~~~~~~~~~~~~~~~~" - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*"); + print_message(CLI_LEVEL_JSON, "*" + std::string(134, '~') + "*"); } print_message(CLI_LEVEL_JSON, ""); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 48b3d78..58a348a 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -386,6 +386,7 @@ TEST(testCapioClEngine, testHomeNode) { capiocl::Engine engine; engine.newFile("A"); EXPECT_TRUE(engine.getHomeNode("A") == nodename); + EXPECT_TRUE(engine.getHomeNode("B") == nodename); } TEST(testCapioClEngine, testInsertFileDependencies) { @@ -399,6 +400,56 @@ TEST(testCapioClEngine, testInsertFileDependencies) { EXPECT_TRUE(engine.getCommitOnFileDependencies("test.txt")[2] == "c"); } +TEST(testCapioClEngine, testEqualDifferentOperator) { + capiocl::Engine engine1, engine2; + + engine1.newFile("A"); + engine2.newFile("A"); + + engine1.setCommitRule("A", capiocl::COMMITTED_ON_CLOSE); + engine2.setCommitRule("A", capiocl::COMMITTED_ON_TERMINATION); + EXPECT_FALSE(engine1 == engine2); + engine2.setCommitRule("A", engine1.getCommitRule("A")); + + engine1.setFireRule("A", capiocl::MODE_NO_UPDATE); + engine2.setFireRule("A", capiocl::MODE_UPDATE); + EXPECT_FALSE(engine1 == engine2); + engine2.setFireRule("A", engine1.getFireRule("A")); + + engine1.setPermanent("A", true); + engine2.setPermanent("A", false); + EXPECT_FALSE(engine1 == engine2); + engine2.setPermanent("A", engine1.isPermanent("A")); + + engine1.setExclude("A", true); + engine2.setExclude("A", false); + EXPECT_FALSE(engine1 == engine2); + engine2.setExclude("A", engine1.isExcluded("A")); + + engine1.setFile("A"); + engine2.setDirectory("A"); + EXPECT_FALSE(engine1 == engine2); + engine2.setFile("A"); + + engine1.setCommitedCloseNumber("A", 10); + engine2.setCommitedCloseNumber("A", 5); + EXPECT_FALSE(engine1 == engine2); + engine2.setCommitedCloseNumber("A", engine1.getCommitCloseCount("A")); + + engine1.setDirectoryFileCount("A", 10); + engine2.setDirectoryFileCount("A", 5); + EXPECT_FALSE(engine1 == engine2); + engine2.setDirectoryFileCount("A", engine1.getDirectoryFileCount("A")); + + engine1.setStoreFileInFileSystem("A"); + engine2.setStoreFileInMemory("A"); + EXPECT_FALSE(engine1 == engine2); + engine2.setStoreFileInFileSystem("A"); + + engine2.newFile("C"); + EXPECT_FALSE(engine1 == engine2); +} + TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { const std::filesystem::path path("./config.json"); const std::string workflow_name = "demo"; @@ -435,6 +486,7 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); EXPECT_TRUE(wf_name == workflow_name); + capiocl::print_message("", ""); EXPECT_TRUE(engine == *new_engine); std::filesystem::remove(path); From 9a755f632b9a8c6fc8931f6db2149a88a3048f66 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 10:59:46 +0200 Subject: [PATCH 12/18] Added tests --- src/Engine.cpp | 16 ++++++++++------ tests/cpp/main.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/Engine.cpp b/src/Engine.cpp index 97b66fb..ca7e1c0 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -154,8 +154,8 @@ void capiocl::Engine::newFile(const std::string &path) { std::string matchKey; size_t matchSize = 0; for (const auto &[filename, data] : _locations) { - if (fnmatch(filename.c_str(), path.c_str(), FNM_PATHNAME) == 0 && - filename.length() > matchSize) { + const bool match = fnmatch(filename.c_str(), path.c_str(), FNM_PATHNAME) == 0; + if (match && filename.length() > matchSize) { LOG("Found match with %s", filename.c_str()); matchSize = filename.length(); matchKey = filename; @@ -233,7 +233,11 @@ void capiocl::Engine::addFileDependency(const std::string &path, std::string &fi file_dependency.erase(remove_if(file_dependency.begin(), file_dependency.end(), isspace), file_dependency.end()); if (const auto itm = _locations.find(path); itm != _locations.end()) { - std::get<9>(itm->second).emplace_back(file_dependency); + if (auto vec = std::get<9>(itm->second); + std::find(vec.begin(), vec.end(), file_dependency) == vec.end()) { + std::get<9>(itm->second).emplace_back(file_dependency); + } + return; } this->newFile(path); @@ -614,7 +618,7 @@ bool capiocl::Engine::operator==(const capiocl::Engine &other) const { // check for producer vector auto this_producer = std::get<0>(this_itm); - auto other_producer = std::get<0>(this_itm); + auto other_producer = std::get<0>(other_itm); if (this_producer.size() != other_producer.size()) { return false; } @@ -627,7 +631,7 @@ bool capiocl::Engine::operator==(const capiocl::Engine &other) const { // check for consumer vector auto this_consumer = std::get<1>(this_itm); - auto other_consumer = std::get<1>(this_itm); + auto other_consumer = std::get<1>(other_itm); if (this_consumer.size() != other_consumer.size()) { return false; } @@ -641,7 +645,7 @@ bool capiocl::Engine::operator==(const capiocl::Engine &other) const { // check for file dependencies auto this_deps = std::get<9>(this_itm); - auto other_deps = std::get<9>(this_itm); + auto other_deps = std::get<9>(other_itm); if (this_deps.size() != other_deps.size()) { return false; } diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 58a348a..09a85e5 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -450,6 +450,50 @@ TEST(testCapioClEngine, testEqualDifferentOperator) { EXPECT_FALSE(engine1 == engine2); } +TEST(testCapioClEngine, testEqualDifferentOperatorDifferentPathSameSize) { + capiocl::Engine engine1, engine2; + engine1.newFile("A"); + engine2.newFile("B"); + EXPECT_FALSE(engine1 == engine2); +} + +TEST(testCapioClEngine, testEqualDifferentProducers) { + capiocl::Engine engine1, engine2; + engine1.newFile("A"); + engine2.newFile("A"); + std::string stepA = "prod1"; + std::string stepB = "prod2"; + engine1.addProducer("A", stepA); + EXPECT_FALSE(engine1 == engine2); + engine2.addProducer("A", stepB); + EXPECT_FALSE(engine1 == engine2); + + engine1.addProducer("A", stepB); + engine2.addProducer("A", stepA); + + engine1.addConsumer("A", stepA); + EXPECT_FALSE(engine1 == engine2); + engine2.addConsumer("A", stepB); + EXPECT_FALSE(engine1 == engine2); +} + +TEST(testCapioClEngine, testFileDependenciesDifferences) { + capiocl::Engine engine1, engine2; + engine1.newFile("A"); + engine2.newFile("A"); + std::string depsA = "prod1"; + std::string depsB = "prod2"; + engine1.addFileDependency("A", depsA); + EXPECT_FALSE(engine1 == engine2); + engine2.addFileDependency("A", depsB); + EXPECT_FALSE(engine1 == engine2); + + engine1.addFileDependency("A", depsB); + EXPECT_FALSE(engine1 == engine2); + engine2.addFileDependency("A", depsA); + EXPECT_TRUE(engine1 == engine2); +} + TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { const std::filesystem::path path("./config.json"); const std::string workflow_name = "demo"; From e1cf7f7a0bb8fd92893071a4a8d2ae9974a3ee3c Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 11:41:24 +0200 Subject: [PATCH 13/18] Tests on serializer and bugfixes --- capiocl.hpp | 2 +- src/Serializer.cpp | 113 ++++++++++++++++----------------------------- tests/cpp/main.cpp | 26 +++++++++++ 3 files changed, 68 insertions(+), 73 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index c946082..c454a9b 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -449,7 +449,7 @@ class Serializer { * @param workflow_name Name of the current workflow * @param filename path of output file @param filename */ - static void dump(const Engine &engine, const std::string workflow_name, + static void dump(const Engine &engine, const std::string &workflow_name, const std::filesystem::path &filename); }; } // namespace capiocl diff --git a/src/Serializer.cpp b/src/Serializer.cpp index bc977ea..172794a 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -3,34 +3,30 @@ #include #include -void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string workflow_name, +void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string &workflow_name, const std::filesystem::path &filename) { START_LOG(gettid(), "call(output='%s')", target.c_str()); nlohmann::json doc; doc["name"] = workflow_name; - // Retrieve the files map - const auto *files = engine.getLocations(); // adjust if it's a different getter + const auto *files = engine.getLocations(); - // Build helper maps: app → inputs / outputs std::unordered_map> app_inputs; std::unordered_map> app_outputs; - // For permanent/exclude/storage std::vector permanent; std::vector exclude; std::vector memory_storage; std::vector fs_storage; + nlohmann::json storage; nlohmann::json io_graph = nlohmann::json::array(); - // We'll also need a mapping of each file → its metadata for streaming reconstruction for (const auto &[path, data] : *files) { const auto &[producers, consumers, commit_rule, fire_rule, permanent_flag, excluded_flag, is_file, n_close, n_dir_files, file_deps, store_in_memory] = data; - // Collect permanent/exclude info if (permanent_flag) { permanent.push_back(path); } @@ -38,14 +34,8 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string exclude.push_back(path); } - // Collect storage info - if (store_in_memory) { - memory_storage.push_back(path); - } else { - fs_storage.push_back(path); - } + store_in_memory ? memory_storage.push_back(path) : fs_storage.push_back(path); - // Collect app relationships for (const auto &p : producers) { app_outputs[p].push_back(path); } @@ -54,20 +44,8 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string } } - // Construct IO_Graph section for (const auto &[app_name, outputs] : app_outputs) { nlohmann::json app; - app["name"] = app_name; - - if (app_inputs.count(app_name)) { - app["input_stream"] = app_inputs[app_name]; - } else { - app["input_stream"] = nlohmann::json::array(); - } - - app["output_stream"] = outputs; - - // ---- streaming ---- nlohmann::json streaming = nlohmann::json::array(); for (const auto &path : outputs) { @@ -76,69 +54,60 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string excluded_flag, is_file, n_close, n_dir_files, file_deps, store_in_fs] = data; - nlohmann::json sitem; - if (is_file) { - sitem["name"] = nlohmann::json::array({path}); - } else { - sitem["dirname"] = nlohmann::json::array({path}); - } + nlohmann::json streaming_item; + std::string committed = commit_rule; + const auto name_kind = is_file ? "name" : "dirname"; + streaming_item[name_kind] = {path}; // Commit rule - std::string committed = commit_rule; if (commit_rule == capiocl::COMMITTED_ON_CLOSE && n_close > 0) { committed += ":" + std::to_string(n_close); - } else if (commit_rule == capiocl::COMMITTED_N_FILES && n_dir_files > 0) { + } + if (commit_rule == capiocl::COMMITTED_N_FILES && n_dir_files > 0) { committed += ":" + std::to_string(n_dir_files); } - sitem["committed"] = committed; - - // Mode / fire rule - sitem["mode"] = fire_rule; + streaming_item["file_deps"] = file_deps; + streaming_item["committed"] = committed; + streaming_item["mode"] = fire_rule; + streaming_item["n_files"] = n_dir_files; - // File dependencies (for COMMITTED_ON_FILE) - if (commit_rule == capiocl::COMMITTED_ON_FILE && !file_deps.empty()) { - sitem["file_deps"] = file_deps; - } - - // Directory file count - if (n_dir_files > 0) { - sitem["n_files"] = n_dir_files; - } - - streaming.push_back(sitem); + streaming.push_back(streaming_item); } - if (!streaming.empty()) { - app["streaming"] = streaming; - } + app["name"] = app_name; + app["input_stream"] = app_inputs[app_name]; + app["output_stream"] = outputs; + app["streaming"] = streaming; io_graph.push_back(app); } - doc["IO_Graph"] = io_graph; - - // ---- permanent / exclude ---- - if (!permanent.empty()) { - doc["permanent"] = permanent; - } - if (!exclude.empty()) { - doc["exclude"] = exclude; + /// Check for app names that have a output_stream empty + for (const auto &[app_name, inputs] : app_inputs) { + bool contained = false; + for (const auto &entry : io_graph) { + contained = entry["name"] == app_name; + if (contained) { + break; + } + } + if (!contained) { + nlohmann::json app; + app["name"] = app_name; + app["input_stream"] = inputs; + app["output_stream"] = nlohmann::json::array(); + io_graph.push_back(app); + } } - // ---- storage ---- - nlohmann::json storage; - if (!memory_storage.empty()) { - storage["memory"] = memory_storage; - } - if (!fs_storage.empty()) { - storage["fs"] = fs_storage; - } - if (!storage.empty()) { - doc["storage"] = storage; - } + doc["IO_Graph"] = io_graph; + doc["permanent"] = permanent; + doc["exclude"] = exclude; + storage["memory"] = memory_storage; + storage["fs"] = fs_storage; + doc["storage"] = storage; - // ---- Write JSON ---- std::ofstream out(filename); if (!out.is_open()) { std::string msg = "Failed to open output file: " + filename.string(); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 09a85e5..7d79d6f 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -535,3 +535,29 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { std::filesystem::remove(path); } + +TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1NcloseNfiles) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "my_command_history.txt"; + std::string producer_name = "_first", consumer_name = "_last"; + + capiocl::Engine engine; + + engine.setDirectory(file_1_name); + engine.setDirectoryFileCount(file_1_name, 10); + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, consumer_name); + + capiocl::Serializer::dump(engine, workflow_name, path); + + std::filesystem::path resolve = ""; + auto [wf_name, new_engine] = capiocl::Parser::parse(path, resolve); + + EXPECT_TRUE(wf_name == workflow_name); + capiocl::print_message("", ""); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); +} \ No newline at end of file From 3eca85fefab966bce9a69aec4775e41115546c28 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 14:44:16 +0200 Subject: [PATCH 14/18] Added failure tests for capiocl::parser --- CMakeLists.txt | 27 +++++++++-- capiocl.hpp | 28 ++++++----- src/Parser.cpp | 98 +++++++++++++++++--------------------- src/Serializer.cpp | 8 +--- tests/cpp/main.cpp | 41 ++++++++++++++++ tests/jsons/V1_test1.json | 1 + tests/jsons/V1_test10.json | 12 +++++ tests/jsons/V1_test11.json | 17 +++++++ tests/jsons/V1_test12.json | 19 ++++++++ tests/jsons/V1_test13.json | 22 +++++++++ tests/jsons/V1_test14.json | 22 +++++++++ tests/jsons/V1_test15.json | 22 +++++++++ tests/jsons/V1_test16.json | 22 +++++++++ tests/jsons/V1_test17.json | 23 +++++++++ tests/jsons/V1_test18.json | 23 +++++++++ tests/jsons/V1_test2.json | 3 ++ tests/jsons/V1_test3.json | 3 ++ tests/jsons/V1_test4.json | 4 ++ tests/jsons/V1_test5.json | 6 +++ tests/jsons/V1_test6.json | 10 ++++ tests/jsons/V1_test7.json | 8 ++++ tests/jsons/V1_test8.json | 9 ++++ tests/jsons/V1_test9.json | 9 ++++ 23 files changed, 360 insertions(+), 77 deletions(-) create mode 100644 tests/jsons/V1_test1.json create mode 100644 tests/jsons/V1_test10.json create mode 100644 tests/jsons/V1_test11.json create mode 100644 tests/jsons/V1_test12.json create mode 100644 tests/jsons/V1_test13.json create mode 100644 tests/jsons/V1_test14.json create mode 100644 tests/jsons/V1_test15.json create mode 100644 tests/jsons/V1_test16.json create mode 100644 tests/jsons/V1_test17.json create mode 100644 tests/jsons/V1_test18.json create mode 100644 tests/jsons/V1_test2.json create mode 100644 tests/jsons/V1_test3.json create mode 100644 tests/jsons/V1_test4.json create mode 100644 tests/jsons/V1_test5.json create mode 100644 tests/jsons/V1_test6.json create mode 100644 tests/jsons/V1_test7.json create mode 100644 tests/jsons/V1_test8.json create mode 100644 tests/jsons/V1_test9.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 78890f4..f3012c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,15 +34,15 @@ if (NOT PROJECT_IS_TOP_LEVEL) message(STATUS "CAPIO-CL included as subproject → tests disabled") endif () -if(CMAKE_BUILD_TYPE STREQUAL "Debug") +if (CMAKE_BUILD_TYPE STREQUAL "Debug") add_compile_options(-O0 -g) -endif() +endif () -if(ENABLE_COVERAGE) +if (ENABLE_COVERAGE) message(STATUS "Building with code coverage instrumentation") add_compile_options(--coverage -O0 -g) add_link_options(--coverage) -endif() +endif () ##################################### # External projects @@ -142,6 +142,19 @@ if (CAPIO_CL_BUILD_TESTS) include(GoogleTest) gtest_discover_tests(CAPIO_CL_tests) + ##################################### + # Copy JSON test files + ##################################### + set(TEST_JSON_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/jsons") + set(TEST_OUTPUT_DIR "/tmp/capio_cl_jsons") + + add_custom_command( + TARGET CAPIO_CL_tests PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_OUTPUT_DIR} + COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_JSON_DIR} ${TEST_OUTPUT_DIR} + COMMENT "Copying JSON test files to build directory" + ) + ##################################### # Install rules ##################################### @@ -149,4 +162,10 @@ if (CAPIO_CL_BUILD_TESTS) install(TARGETS CAPIO_CL_tests RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) + + # Copy tests jsons + install(DIRECTORY ${TEST_JSON_DIR} + DESTINATION ${CMAKE_INSTALL_BINDIR}/jsons + FILES_MATCHING PATTERN "*.json" + ) endif () diff --git a/capiocl.hpp b/capiocl.hpp index c454a9b..c5292a5 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -414,17 +414,6 @@ class Parser { */ static bool isInteger(const std::string &s); - /** - * @brief compare two paths - * - * @param path - * @param base - * @return true if @p path is a subdirectory of base - * @return false otherwise - */ - static inline bool firstIsSubpathOfSecond(const std::filesystem::path &path, - const std::filesystem::path &base); - public: /** * @brief Perform the parsing of the capio_server configuration file @@ -436,8 +425,21 @@ class Parser { * the config file */ static std::tuple parse(const std::filesystem::path &source, - std::filesystem::path &resolve_prexix, - bool store_only_in_memory = false); + std::filesystem::path resolve_prexix = "", + bool store_only_in_memory = false); +}; + +/** + * Custom exception for errors occurring within the Parser component + */ +class ParserException : public std::exception { + std::string message; + + public: + explicit ParserException(const std::string &msg) : message(msg) { + print_message(CLI_LEVEL_ERROR, msg); + } + const char *what() const noexcept override { return message.c_str(); } }; class Serializer { diff --git a/src/Parser.cpp b/src/Parser.cpp index b5e6193..c6c0213 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -14,14 +14,8 @@ bool capiocl::Parser::isInteger(const std::string &s) { return res; }; -bool capiocl::Parser::firstIsSubpathOfSecond(const std::filesystem::path &path, - const std::filesystem::path &base) { - const auto mismatch_pair = std::mismatch(path.begin(), path.end(), base.begin(), base.end()); - return mismatch_pair.second == base.end(); -} - std::tuple -capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path &resolve_prefix, +capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path resolve_prefix, bool store_only_in_memory) { bool skip_resolve = resolve_prefix.empty(); @@ -30,31 +24,24 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat START_LOG(gettid(), "call(config_file='%s')", source.c_str()); if (source.empty()) { - return {workflow_name, locations}; + throw capiocl::ParserException("Empty source file name!"); } // ---- Load JSON ---- std::ifstream file(source); if (!file.is_open()) { - std::string msg = "Failed to open config file: " + source.string(); - print_message(CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); + throw capiocl::ParserException("Failed to open file!"); } nlohmann::json doc; - try { - file >> doc; - } catch (const std::exception &e) { - std::string err = "JSON parse error: "; - err += e.what(); - print_message(CLI_LEVEL_ERROR, err); - ERR_EXIT(err.c_str()); - } + file >> doc; // ---- workflow name ---- - if (!doc.contains("name") || !doc["name"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Missing workflow name!"); - ERR_EXIT("Error: workflow name is mandatory"); + if (!doc.contains("name")) { + throw capiocl::ParserException("Missing workflow name!"); + } + if (!doc["name"].is_string()) { + throw capiocl::ParserException("Wrong data type for workflow name!"); } workflow_name = doc["name"].get(); @@ -62,14 +49,19 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat LOG("Parsing configuration for workflow: %s", workflow_name.c_str()); // ---- IO_Graph ---- - if (!doc.contains("IO_Graph") || !doc["IO_Graph"].is_array()) { - ERR_EXIT("Error: IO_Graph section missing or invalid"); + if (!doc.contains("IO_Graph")) { + throw capiocl::ParserException("Missing IO_Graph section"); + } + if (!doc["IO_Graph"].is_array()) { + throw capiocl::ParserException("Wrong data type for IO_Graph section"); } for (const auto &app : doc["IO_Graph"]) { - if (!app.contains("name") || !app["name"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Missing IO_Graph name or name is not a valid string!"); - ERR_EXIT("Error: app name is mandatory"); + if (!app.contains("name")) { + throw capiocl::ParserException("Missing name for streaming item!"); + } + if (!app["name"].is_string()) { + throw capiocl::ParserException("Wrong type for name streaming entry!"); } std::string app_name = app["name"].get(); @@ -77,10 +69,12 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat LOG("Parsing config for app %s", app_name.c_str()); // ---- input_stream ---- - if (!app.contains("input_stream") || !app["input_stream"].is_array()) { - std::string msg = "No input_stream section found for app " + app_name; - print_message(CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); + if (!app.contains("input_stream")) { + throw capiocl::ParserException("No input_stream section found for app " + app_name); + } + if (!app["input_stream"].is_array()) { + throw capiocl::ParserException("input_stream section for app " + app_name + + " is not array!"); } print_message(CLI_LEVEL_JSON, "Parsing input_stream for app " + app_name); @@ -96,11 +90,14 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat } // ---- output_stream ---- - if (!app.contains("output_stream") || !app["output_stream"].is_array()) { - std::string msg = "No output_stream section found for app " + app_name; - print_message(CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); + if (!app.contains("output_stream")) { + throw capiocl::ParserException("No output_stream section found for app " + app_name); } + if (!app["output_stream"].is_array()) { + throw capiocl::ParserException("output_stream section for app " + app_name + + " is not array!"); + } + print_message(CLI_LEVEL_JSON, "Parsing output_stream for app " + app_name); for (const auto &itm : app["output_stream"]) { std::filesystem::path file_path(itm.get()); @@ -143,18 +140,15 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat streaming_names.push_back(p); } } else { - print_message( - CLI_LEVEL_ERROR, + throw capiocl::ParserException( "Missing streaming name/dirname, or name/dirname is not an array for app " + - app_name); - ERR_EXIT("error: either name or dirname in streaming section is required"); + app_name); } // Commit rule. Optional in nature, hence no check required! if (stream_item.contains("committed")) { if (!stream_item["committed"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Error: invalid type for commit rule!"); - ERR_EXIT("Error: invalid type for commit rule!"); + throw capiocl::ParserException("Error: invalid type for commit rule!"); } std::string committed = stream_item["committed"].get(); @@ -163,14 +157,15 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat commit_rule = committed.substr(0, pos); std::string count_str = committed.substr(pos + 1); if (!isInteger(count_str)) { - ERR_EXIT("invalid number in commit rule"); + throw capiocl::ParserException( + "commit rule argument is not an integer!"); } if (commit_rule == COMMITTED_ON_CLOSE) { n_close = std::stol(count_str); } else if (commit_rule == COMMITTED_N_FILES) { n_files = std::stol(count_str); } else { - ERR_EXIT("invalid commit rule type"); + throw capiocl::ParserException("Invalid commit rule!"); } } else { commit_rule = committed; @@ -180,7 +175,8 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat if (commit_rule == COMMITTED_ON_FILE) { if (!stream_item.contains("file_deps") || !stream_item["file_deps"].is_array()) { - ERR_EXIT("commit rule is on_file but no file_deps section found"); + throw capiocl::ParserException( + "commit rule is on_file but no file_deps section found"); } for (const auto &dep : stream_item["file_deps"]) { std::filesystem::path p(dep.get()); @@ -196,25 +192,21 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat commit_rule != capiocl::COMMITTED_ON_CLOSE && commit_rule != capiocl::COMMITTED_ON_FILE && commit_rule != capiocl::COMMITTED_ON_TERMINATION) { - print_message(CLI_LEVEL_ERROR, "Error: commit rule " + commit_rule + - " is not one of the allowed one!"); - ERR_EXIT("Unknown commit rule %s", commit_rule.c_str()); + throw capiocl::ParserException("Error: commit rule " + commit_rule + + " is not one of the allowed one!"); } } // Firing rule. Optional in nature, hence no check required! if (stream_item.contains("mode")) { if (!stream_item["mode"].is_string()) { - print_message(CLI_LEVEL_ERROR, "Error: invalid type for mode"); - ERR_EXIT("Error: invalid type for mode"); + throw capiocl::ParserException("Error: invalid firing rule data type"); } mode = stream_item["mode"].get(); if (mode != capiocl::MODE_UPDATE && mode != capiocl::MODE_NO_UPDATE) { - print_message(CLI_LEVEL_ERROR, - "Error: invalid firing rule provided for app: " + app_name); - ERR_EXIT("Error: invalid firing rule provided for app: %s", - app_name.c_str()); + throw capiocl::ParserException( + "Error: fire rule is not one of the allowed ones for app: " + app_name); } } diff --git a/src/Serializer.cpp b/src/Serializer.cpp index 172794a..d12704c 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -109,13 +109,7 @@ void capiocl::Serializer::dump(const capiocl::Engine &engine, const std::string doc["storage"] = storage; std::ofstream out(filename); - if (!out.is_open()) { - std::string msg = "Failed to open output file: " + filename.string(); - capiocl::print_message(capiocl::CLI_LEVEL_ERROR, msg); - ERR_EXIT(msg.c_str()); - } - - out << std::setw(4) << doc << std::endl; + out << std::setw(2) << doc << std::endl; capiocl::print_message(capiocl::CLI_LEVEL_INFO, "Configuration serialized to " + filename.string()); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 7d79d6f..e6bdca6 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -560,4 +560,45 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1NcloseNfiles) { EXPECT_TRUE(engine == *new_engine); std::filesystem::remove(path); +} + +TEST(testCapioSerializerParser, testParserException) { + std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons"; + capiocl::print_message(capiocl::CLI_LEVEL_INFO, "Loading jsons from " + JSON_DIR.string()); + bool exception_catched = false; + + std::vector test_filenames = { + "", + "ANonExistingFile", + JSON_DIR / "V1_test1.json", + JSON_DIR / "V1_test2.json", + JSON_DIR / "V1_test3.json", + JSON_DIR / "V1_test4.json", + JSON_DIR / "V1_test5.json", + JSON_DIR / "V1_test6.json", + JSON_DIR / "V1_test7.json", + JSON_DIR / "V1_test8.json", + JSON_DIR / "V1_test9.json", + JSON_DIR / "V1_test10.json", + JSON_DIR / "V1_test11.json", + JSON_DIR / "V1_test12.json", + JSON_DIR / "V1_test13.json", + JSON_DIR / "V1_test14.json", + JSON_DIR / "V1_test15.json", + JSON_DIR / "V1_test16.json", + JSON_DIR / "V1_test17.json", + JSON_DIR / "V1_test18.json", + }; + + for (const auto &test : test_filenames) { + exception_catched = false; + try { + capiocl::print_message(capiocl::CLI_LEVEL_WARNING, "Testing on file " + test.string()); + auto [wf_name, engine] = capiocl::Parser::parse(test); + } catch (capiocl::ParserException &e) { + exception_catched = true; + } + EXPECT_TRUE(exception_catched); + capiocl::print_message(capiocl::CLI_LEVEL_WARNING); + } } \ No newline at end of file diff --git a/tests/jsons/V1_test1.json b/tests/jsons/V1_test1.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/tests/jsons/V1_test1.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tests/jsons/V1_test10.json b/tests/jsons/V1_test10.json new file mode 100644 index 0000000..ec49bcf --- /dev/null +++ b/tests/jsons/V1_test10.json @@ -0,0 +1,12 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test11.json b/tests/jsons/V1_test11.json new file mode 100644 index 0000000..81a1b16 --- /dev/null +++ b/tests/jsons/V1_test11.json @@ -0,0 +1,17 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + {} + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test12.json b/tests/jsons/V1_test12.json new file mode 100644 index 0000000..a0228e7 --- /dev/null +++ b/tests/jsons/V1_test12.json @@ -0,0 +1,19 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test13.json b/tests/jsons/V1_test13.json new file mode 100644 index 0000000..4f17be1 --- /dev/null +++ b/tests/jsons/V1_test13.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test14.json b/tests/jsons/V1_test14.json new file mode 100644 index 0000000..28cd8a2 --- /dev/null +++ b/tests/jsons/V1_test14.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "a:b" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test15.json b/tests/jsons/V1_test15.json new file mode 100644 index 0000000..b9aedd7 --- /dev/null +++ b/tests/jsons/V1_test15.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test16.json b/tests/jsons/V1_test16.json new file mode 100644 index 0000000..9c27d52 --- /dev/null +++ b/tests/jsons/V1_test16.json @@ -0,0 +1,22 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_file" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test17.json b/tests/jsons/V1_test17.json new file mode 100644 index 0000000..acf861b --- /dev/null +++ b/tests/jsons/V1_test17.json @@ -0,0 +1,23 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode" : ["failMe"] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test18.json b/tests/jsons/V1_test18.json new file mode 100644 index 0000000..e0f3445 --- /dev/null +++ b/tests/jsons/V1_test18.json @@ -0,0 +1,23 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode" : "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test2.json b/tests/jsons/V1_test2.json new file mode 100644 index 0000000..2e31866 --- /dev/null +++ b/tests/jsons/V1_test2.json @@ -0,0 +1,3 @@ +{ + "name": [] +} \ No newline at end of file diff --git a/tests/jsons/V1_test3.json b/tests/jsons/V1_test3.json new file mode 100644 index 0000000..71d0d33 --- /dev/null +++ b/tests/jsons/V1_test3.json @@ -0,0 +1,3 @@ +{ + "name" : "test" +} \ No newline at end of file diff --git a/tests/jsons/V1_test4.json b/tests/jsons/V1_test4.json new file mode 100644 index 0000000..90597be --- /dev/null +++ b/tests/jsons/V1_test4.json @@ -0,0 +1,4 @@ +{ + "name" : "test", + "IO_Graph" : "fail" +} \ No newline at end of file diff --git a/tests/jsons/V1_test5.json b/tests/jsons/V1_test5.json new file mode 100644 index 0000000..0230f1a --- /dev/null +++ b/tests/jsons/V1_test5.json @@ -0,0 +1,6 @@ +{ + "name": "test", + "IO_Graph": [ + {} + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test6.json b/tests/jsons/V1_test6.json new file mode 100644 index 0000000..f0c49ea --- /dev/null +++ b/tests/jsons/V1_test6.json @@ -0,0 +1,10 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": [ + "failMe" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test7.json b/tests/jsons/V1_test7.json new file mode 100644 index 0000000..382fe8a --- /dev/null +++ b/tests/jsons/V1_test7.json @@ -0,0 +1,8 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test8.json b/tests/jsons/V1_test8.json new file mode 100644 index 0000000..b647862 --- /dev/null +++ b/tests/jsons/V1_test8.json @@ -0,0 +1,9 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test9.json b/tests/jsons/V1_test9.json new file mode 100644 index 0000000..67cf4f5 --- /dev/null +++ b/tests/jsons/V1_test9.json @@ -0,0 +1,9 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream" : ["file"] + } + ] +} \ No newline at end of file From 8f91bbac94349ae3293764e49281514cd763bc0b Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 15:13:13 +0200 Subject: [PATCH 15/18] Tests --- CMakeLists.txt | 5 ++--- config.json | 38 ++++++++++++++++++++++++++++++++++++++ src/Engine.cpp | 2 +- src/Parser.cpp | 25 +++++++++++++++++-------- tests/cpp/main.cpp | 22 +++++++++++++++++++--- tests/jsons/V1_test19.json | 24 ++++++++++++++++++++++++ tests/jsons/V1_test20.json | 26 ++++++++++++++++++++++++++ tests/jsons/V1_test21.json | 27 +++++++++++++++++++++++++++ 8 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 config.json create mode 100644 tests/jsons/V1_test19.json create mode 100644 tests/jsons/V1_test20.json create mode 100644 tests/jsons/V1_test21.json diff --git a/CMakeLists.txt b/CMakeLists.txt index f3012c1..6f49486 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,12 +146,11 @@ if (CAPIO_CL_BUILD_TESTS) # Copy JSON test files ##################################### set(TEST_JSON_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/jsons") - set(TEST_OUTPUT_DIR "/tmp/capio_cl_jsons") add_custom_command( TARGET CAPIO_CL_tests PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_OUTPUT_DIR} - COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_JSON_DIR} ${TEST_OUTPUT_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "/tmp/capio_cl_jsons" + COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_JSON_DIR} "/tmp/capio_cl_jsons" COMMENT "Copying JSON test files to build directory" ) diff --git a/config.json b/config.json new file mode 100644 index 0000000..66d4ba0 --- /dev/null +++ b/config.json @@ -0,0 +1,38 @@ +{ + "IO_Graph": [ + { + "input_stream": [], + "name": "_first", + "output_stream": [ + "file1.txt" + ], + "streaming": [ + { + "committed": "on_termination", + "dirname": [ + "file1.txt" + ], + "file_deps": [], + "mode": "update", + "n_files": 10 + } + ] + }, + { + "input_stream": [ + "file1.txt" + ], + "name": "_last", + "output_stream": [] + } + ], + "exclude": [], + "name": "demo", + "permanent": [], + "storage": { + "fs": [ + "file1.txt" + ], + "memory": [] + } +} diff --git a/src/Engine.cpp b/src/Engine.cpp index ca7e1c0..408af90 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -488,7 +488,7 @@ long capiocl::Engine::getCommitCloseCount(std::filesystem::path::iterator::refer this->newFile(path); return getCommitCloseCount(path); -}; +} std::vector capiocl::Engine::getCommitOnFileDependencies(const std::filesystem::path &path) { diff --git a/src/Parser.cpp b/src/Parser.cpp index c6c0213..4f51b58 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -12,7 +12,7 @@ bool capiocl::Parser::isInteger(const std::string &s) { res = *p == 0; } return res; -}; +} std::tuple capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path resolve_prefix, @@ -81,8 +81,8 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat for (const auto &itm : app["input_stream"]) { std::filesystem::path file_path(itm.get()); if (file_path.is_relative() && !skip_resolve) { - print_message(CLI_LEVEL_WARNING, - "Path : " + file_path.string() + " IS RELATIVE! resolving..."); + auto msg = "Path : " + file_path.string() + " IS RELATIVE! resolving..."; + print_message(CLI_LEVEL_WARNING, msg); file_path = resolve_prefix / file_path; } locations->newFile(file_path); @@ -102,8 +102,8 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat for (const auto &itm : app["output_stream"]) { std::filesystem::path file_path(itm.get()); if (file_path.is_relative() && !skip_resolve) { - print_message(CLI_LEVEL_WARNING, - "Path : " + file_path.string() + " IS RELATIVE! resolving..."); + auto msg = "Path : " + file_path.string() + " IS RELATIVE! resolving..."; + print_message(CLI_LEVEL_WARNING, msg); file_path = resolve_prefix / file_path; } locations->newFile(file_path); @@ -266,17 +266,26 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat } // ---- storage ---- - if (doc.contains("storage") && doc["storage"].is_object()) { + if (doc.contains("storage")) { + if (!doc["storage"].is_object()) { + throw capiocl::ParserException("Wrong data type for storage section!"); + } const auto &storage = doc["storage"]; - if (storage.contains("memory") && storage["memory"].is_array()) { + if (storage.contains("memory")) { + if (!storage["memory"].is_array()) { + throw capiocl::ParserException("Wrong data type for memory storage section"); + } for (const auto &f : storage["memory"]) { auto file_str = f.get(); locations->setStoreFileInMemory(file_str); } } - if (storage.contains("fs") && storage["fs"].is_array()) { + if (storage.contains("fs")) { + if (!storage["fs"].is_array()) { + throw capiocl::ParserException("Wrong data type for fs storage section"); + } for (const auto &f : storage["fs"]) { auto file_str = f.get(); locations->setStoreFileInFileSystem(file_str); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index e6bdca6..839e8f0 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -1,4 +1,5 @@ #include "capiocl.hpp" +#include #include TEST(testCapioClEngine, testInstantiation) { @@ -533,14 +534,16 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { capiocl::print_message("", ""); EXPECT_TRUE(engine == *new_engine); + auto [wf_name1, new_engine1] = capiocl::Parser::parse(path, resolve, true); + EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), 3); + std::filesystem::remove(path); } TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1NcloseNfiles) { const std::filesystem::path path("./config.json"); const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", - file_3_name = "my_command_history.txt"; + const std::string file_1_name = "file1.txt"; std::string producer_name = "_first", consumer_name = "_last"; capiocl::Engine engine; @@ -562,6 +565,14 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1NcloseNfiles) { std::filesystem::remove(path); } +template std::string demangled_name(const T &obj) { + int status; + const char *mangled = typeid(obj).name(); + std::unique_ptr demangled( + abi::__cxa_demangle(mangled, nullptr, nullptr, &status), std::free); + return status == 0 ? demangled.get() : mangled; +} + TEST(testCapioSerializerParser, testParserException) { std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons"; capiocl::print_message(capiocl::CLI_LEVEL_INFO, "Loading jsons from " + JSON_DIR.string()); @@ -588,6 +599,9 @@ TEST(testCapioSerializerParser, testParserException) { JSON_DIR / "V1_test16.json", JSON_DIR / "V1_test17.json", JSON_DIR / "V1_test18.json", + JSON_DIR / "V1_test19.json", + JSON_DIR / "V1_test20.json", + JSON_DIR / "V1_test21.json", }; for (const auto &test : test_filenames) { @@ -595,8 +609,10 @@ TEST(testCapioSerializerParser, testParserException) { try { capiocl::print_message(capiocl::CLI_LEVEL_WARNING, "Testing on file " + test.string()); auto [wf_name, engine] = capiocl::Parser::parse(test); - } catch (capiocl::ParserException &e) { + } catch (std::exception &e) { exception_catched = true; + EXPECT_TRUE(demangled_name(e) == "capiocl::ParserException"); + EXPECT_GT(std::string(e.what()).size(), 0); } EXPECT_TRUE(exception_catched); capiocl::print_message(capiocl::CLI_LEVEL_WARNING); diff --git a/tests/jsons/V1_test19.json b/tests/jsons/V1_test19.json new file mode 100644 index 0000000..27a3672 --- /dev/null +++ b/tests/jsons/V1_test19.json @@ -0,0 +1,24 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": "failMe" +} \ No newline at end of file diff --git a/tests/jsons/V1_test20.json b/tests/jsons/V1_test20.json new file mode 100644 index 0000000..ed357e9 --- /dev/null +++ b/tests/jsons/V1_test20.json @@ -0,0 +1,26 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1_test21.json b/tests/jsons/V1_test21.json new file mode 100644 index 0000000..fe8a801 --- /dev/null +++ b/tests/jsons/V1_test21.json @@ -0,0 +1,27 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": [], + "fs": "failMe" + } +} \ No newline at end of file From c40020989cb31cc37b3dfc63ec93d2dced9eabbd Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 15:41:03 +0200 Subject: [PATCH 16/18] Added testsg --- capiocl.hpp | 15 +++++---------- src/Parser.cpp | 28 ++++++++++++---------------- tests/cpp/main.cpp | 1 + tests/jsons/V1_test22.json | 24 ++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 26 deletions(-) create mode 100644 tests/jsons/V1_test22.json diff --git a/capiocl.hpp b/capiocl.hpp index c5292a5..d60e667 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -107,10 +107,13 @@ class Engine { * * @param str Input string. * @param n Number of characters to keep from the end. - * @return Truncated string with optional "[..]" prefix. + * @return Truncated string with "[..]" prefix. */ static std::string truncateLastN(const std::string &str, const std::size_t n) { - return str.length() > n ? "[..] " + str.substr(str.length() - n) : str; + if (str.length() > n) { + return "[..] " + str.substr(str.length() - n); + } + return str; } protected: @@ -405,14 +408,6 @@ class Engine { * */ class Parser { - /** - * @brief Check if a string is a representation of a integer number - * - * @param s - * @return true - * @return false - */ - static bool isInteger(const std::string &s); public: /** diff --git a/src/Parser.cpp b/src/Parser.cpp index 4f51b58..b28d060 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -3,17 +3,6 @@ #include #include -bool capiocl::Parser::isInteger(const std::string &s) { - START_LOG(gettid(), "call(%s)", s.c_str()); - bool res = false; - if (!s.empty()) { - char *p; - strtol(s.c_str(), &p, 10); - res = *p == 0; - } - return res; -} - std::tuple capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path resolve_prefix, bool store_only_in_memory) { @@ -156,10 +145,14 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat if (pos != std::string::npos) { commit_rule = committed.substr(0, pos); std::string count_str = committed.substr(pos + 1); - if (!isInteger(count_str)) { - throw capiocl::ParserException( - "commit rule argument is not an integer!"); + try { + size_t num_len; + std::stoi(count_str, &num_len); + } catch (...) { + auto msg = "commit rule argument is not an integer!"; + throw capiocl::ParserException(msg); } + if (commit_rule == COMMITTED_ON_CLOSE) { n_close = std::stol(count_str); } else if (commit_rule == COMMITTED_N_FILES) { @@ -211,12 +204,15 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat } // n_files (optional) - if (stream_item.contains("n_files") && stream_item["n_files"].is_number_integer()) { + if (stream_item.contains("n_files")) { + if (!stream_item["n_files"].is_number_integer()) { + throw capiocl::ParserException("wrong type for n_files!"); + } n_files = stream_item["n_files"].get(); } for (auto &path : streaming_names) { - if (n_files != -1) { + if (n_files != 0) { locations->setDirectoryFileCount(path, n_files); } if (is_file) { diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 839e8f0..b7f24c8 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -602,6 +602,7 @@ TEST(testCapioSerializerParser, testParserException) { JSON_DIR / "V1_test19.json", JSON_DIR / "V1_test20.json", JSON_DIR / "V1_test21.json", + JSON_DIR / "V1_test22.json", }; for (const auto &test : test_filenames) { diff --git a/tests/jsons/V1_test22.json b/tests/jsons/V1_test22.json new file mode 100644 index 0000000..b77b2ba --- /dev/null +++ b/tests/jsons/V1_test22.json @@ -0,0 +1,24 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update", + "n_files" : "failMe" + } + ] + } + ] +} \ No newline at end of file From 900f12d99bf7c3c97ea34c23bcf5796e64c7c8fc Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 16:06:21 +0200 Subject: [PATCH 17/18] Cleanup --- capiocl.hpp | 3 +++ src/Parser.cpp | 52 +++++++++++++++++++++++--------------------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index d60e667..b939eda 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -409,6 +409,9 @@ class Engine { */ class Parser { + static std::filesystem::path resolve(std::filesystem::path path, + const std::filesystem::path &prefix); + public: /** * @brief Perform the parsing of the capio_server configuration file diff --git a/src/Parser.cpp b/src/Parser.cpp index b28d060..8a0a1f5 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -3,11 +3,26 @@ #include #include +std::filesystem::path capiocl::Parser::resolve(std::filesystem::path path, + const std::filesystem::path &prefix) { + if (prefix.empty()) { + return path; + } + + if (path.is_absolute()) { + return path; + } + + auto msg = "Path : " + path.string() + " IS RELATIVE! resolving..."; + print_message(CLI_LEVEL_WARNING, msg); + + return prefix / path; +} + std::tuple capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path resolve_prefix, bool store_only_in_memory) { - bool skip_resolve = resolve_prefix.empty(); std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto locations = new Engine(); START_LOG(gettid(), "call(config_file='%s')", source.c_str()); @@ -68,12 +83,7 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat print_message(CLI_LEVEL_JSON, "Parsing input_stream for app " + app_name); for (const auto &itm : app["input_stream"]) { - std::filesystem::path file_path(itm.get()); - if (file_path.is_relative() && !skip_resolve) { - auto msg = "Path : " + file_path.string() + " IS RELATIVE! resolving..."; - print_message(CLI_LEVEL_WARNING, msg); - file_path = resolve_prefix / file_path; - } + auto file_path = resolve(itm.get(), resolve_prefix); locations->newFile(file_path); locations->addConsumer(file_path, app_name); } @@ -89,12 +99,7 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat print_message(CLI_LEVEL_JSON, "Parsing output_stream for app " + app_name); for (const auto &itm : app["output_stream"]) { - std::filesystem::path file_path(itm.get()); - if (file_path.is_relative() && !skip_resolve) { - auto msg = "Path : " + file_path.string() + " IS RELATIVE! resolving..."; - print_message(CLI_LEVEL_WARNING, msg); - file_path = resolve_prefix / file_path; - } + auto file_path = resolve(itm.get(), resolve_prefix); locations->newFile(file_path); locations->addProducer(file_path, app_name); } @@ -113,20 +118,14 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat // name or dirname if (stream_item.contains("name") && stream_item["name"].is_array()) { for (const auto &nm : stream_item["name"]) { - std::filesystem::path p(nm.get()); - if (p.is_relative() && !skip_resolve) { - p = resolve_prefix / p; - } - streaming_names.push_back(p); + auto nm_resolved = resolve(nm.get(), resolve_prefix); + streaming_names.push_back(nm_resolved); } } else if (stream_item.contains("dirname") && stream_item["dirname"].is_array()) { is_file = false; for (const auto &nm : stream_item["dirname"]) { - std::filesystem::path p(nm.get()); - if (p.is_relative() && !skip_resolve) { - p = resolve_prefix / p; - } - streaming_names.push_back(p); + auto nm_resolved = resolve(nm.get(), resolve_prefix); + streaming_names.push_back(nm_resolved); } } else { throw capiocl::ParserException( @@ -172,11 +171,8 @@ capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::pat "commit rule is on_file but no file_deps section found"); } for (const auto &dep : stream_item["file_deps"]) { - std::filesystem::path p(dep.get()); - if (p.is_relative() && !skip_resolve) { - p = resolve_prefix / p; - } - file_deps.push_back(p); + auto dep_resolved = resolve(dep.get(), resolve_prefix); + file_deps.push_back(dep_resolved); } } From 78098797295a66e446ab113e7eaab925bee54de1 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 9 Oct 2025 16:33:52 +0200 Subject: [PATCH 18/18] Added tests --- capiocl.hpp | 6 +++--- config.json | 38 -------------------------------------- src/Parser.cpp | 9 +++++---- tests/cpp/main.cpp | 21 +++++++++++++++++++-- tests/jsons/V1_test0.json | 25 +++++++++++++++++++++++++ tests/jsons/V1_test23.json | 23 +++++++++++++++++++++++ 6 files changed, 75 insertions(+), 47 deletions(-) delete mode 100644 config.json create mode 100644 tests/jsons/V1_test0.json create mode 100644 tests/jsons/V1_test23.json diff --git a/capiocl.hpp b/capiocl.hpp index b939eda..0dcf075 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -417,14 +417,14 @@ class Parser { * @brief Perform the parsing of the capio_server configuration file * * @param source - * @param resolve_prexix + * @param resolve_prefix * @param store_only_in_memory Set to true to set all files to be stored in memory * @return Tuple with workflow name and CapioCLEngine instance with the information provided by * the config file */ static std::tuple parse(const std::filesystem::path &source, - std::filesystem::path resolve_prexix = "", - bool store_only_in_memory = false); + const std::filesystem::path &resolve_prefix = "", + bool store_only_in_memory = false); }; /** diff --git a/config.json b/config.json deleted file mode 100644 index 66d4ba0..0000000 --- a/config.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "IO_Graph": [ - { - "input_stream": [], - "name": "_first", - "output_stream": [ - "file1.txt" - ], - "streaming": [ - { - "committed": "on_termination", - "dirname": [ - "file1.txt" - ], - "file_deps": [], - "mode": "update", - "n_files": 10 - } - ] - }, - { - "input_stream": [ - "file1.txt" - ], - "name": "_last", - "output_stream": [] - } - ], - "exclude": [], - "name": "demo", - "permanent": [], - "storage": { - "fs": [ - "file1.txt" - ], - "memory": [] - } -} diff --git a/src/Parser.cpp b/src/Parser.cpp index 8a0a1f5..0440802 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -13,15 +13,16 @@ std::filesystem::path capiocl::Parser::resolve(std::filesystem::path path, return path; } - auto msg = "Path : " + path.string() + " IS RELATIVE! resolving..."; + auto resolved = prefix / path; + const auto msg = "Path : " + path.string() + " IS RELATIVE! Resolved to: " + resolved.string(); print_message(CLI_LEVEL_WARNING, msg); - return prefix / path; + return resolved; } std::tuple -capiocl::Parser::parse(const std::filesystem::path &source, std::filesystem::path resolve_prefix, - bool store_only_in_memory) { +capiocl::Parser::parse(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto locations = new Engine(); diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index b7f24c8..0b2af72 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -499,7 +499,7 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { const std::filesystem::path path("./config.json"); const std::string workflow_name = "demo"; const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", - file_3_name = "my_command_history.txt"; + file_3_name = "my_command_history.txt", file_4_name = "/tmp"; std::string producer_name = "_first", consumer_name = "_last", intermediate_name = "_middle"; capiocl::Engine engine; @@ -523,6 +523,12 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { engine.addProducer(file_3_name, consumer_name); engine.addProducer(file_3_name, intermediate_name); engine.setExclude(file_3_name, true); + + engine.setCommitRule(file_4_name, capiocl::COMMITTED_N_FILES); + engine.setFireRule(file_4_name, capiocl::MODE_NO_UPDATE); + engine.setDirectoryFileCount(file_4_name, 10); + engine.addProducer(file_4_name, intermediate_name); + engine.print(); capiocl::Serializer::dump(engine, workflow_name, path); @@ -535,7 +541,7 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1) { EXPECT_TRUE(engine == *new_engine); auto [wf_name1, new_engine1] = capiocl::Parser::parse(path, resolve, true); - EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), 3); + EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); std::filesystem::remove(path); } @@ -565,6 +571,16 @@ TEST(testCapioSerializerParser, testSerializeParseCAPIOCLV1NcloseNfiles) { std::filesystem::remove(path); } +TEST(testCapioSerializerParser, testParserResolveAbsolute) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test0.json"); + auto [wf_name, engine] = capiocl::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(wf_name == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + EXPECT_TRUE(engine->contains("/tmp/file2")); + EXPECT_TRUE(engine->contains("/tmp/file3")); +} + template std::string demangled_name(const T &obj) { int status; const char *mangled = typeid(obj).name(); @@ -603,6 +619,7 @@ TEST(testCapioSerializerParser, testParserException) { JSON_DIR / "V1_test20.json", JSON_DIR / "V1_test21.json", JSON_DIR / "V1_test22.json", + JSON_DIR / "V1_test23.json", }; for (const auto &test : test_filenames) { diff --git a/tests/jsons/V1_test0.json b/tests/jsons/V1_test0.json new file mode 100644 index 0000000..e6ed298 --- /dev/null +++ b/tests/jsons/V1_test0.json @@ -0,0 +1,25 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "/tmp/file", + "file3" + ], + "output_stream": [ + "file1", + "file2" + ], + "streaming": [ + { + "name": [ + "/tmp/file" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_test23.json b/tests/jsons/V1_test23.json new file mode 100644 index 0000000..6f223d4 --- /dev/null +++ b/tests/jsons/V1_test23.json @@ -0,0 +1,23 @@ +{ + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "n_files:failMe", + "mode": "no_update" + } + ] + } + ] +} \ No newline at end of file