Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
167 commits
Select commit Hold shift + click to select a range
cc96736
Cleanup of CAPIOV1 unused code
Jul 17, 2024
00ac7d9
storage_engine for metadata handling
Jul 18, 2024
4ebc302
Improved systemcall handlers
Jul 19, 2024
034eed9
Support for commit rule
Jul 30, 2024
18ebd6f
Implemented write cache for write requests
Jul 31, 2024
c9ea4f9
Filesystem Monitor thread
Aug 2, 2024
4f209b1
Improved read request cache
Aug 2, 2024
61d23b0
Capioctl
Aug 10, 2024
5627e72
Refactor code to reflect choosen architecture
Sep 5, 2024
468f583
JSON parser
Sep 5, 2024
188f8a5
Improved logging request
Sep 10, 2024
1b035b9
Varius bugfixes
Sep 10, 2024
6af3b46
cache
Sep 11, 2024
a1558ab
File System Monitor fixes
Sep 11, 2024
59b88b1
Memory initialization and bugfixes
Sep 11, 2024
60aae95
Code cleanup and bugfixes
Sep 12, 2024
152b219
Support for n_files
Sep 12, 2024
e6d0c10
Support for CoF
marcoSanti Sep 17, 2024
aa2824c
Support for CoC:n
Sep 21, 2024
8a81ade
Bugfixes
Sep 27, 2024
0d1d793
Moved CAPIO metadata to other directory
Oct 1, 2024
8c48981
Protected map access to avoid std::out_of_range
Oct 2, 2024
cc23d15
updated readme
Oct 2, 2024
d42b14d
Merge pull request #112 from marcoSanti/master
marcoSanti Oct 7, 2024
17df088
Added doxygen documentation
Oct 7, 2024
85e293e
Added cache to consent to proceed handler
Oct 7, 2024
796208f
Merge pull request #114 from High-Performance-IO/consent_cache
marcoSanti Oct 7, 2024
ddacc35
Improvements in code cleaness and performance
Oct 7, 2024
0d916e0
Improved performances
Oct 8, 2024
a1253cd
added cache to commit files
Oct 10, 2024
0cfd470
Improved code
Oct 10, 2024
eff20d7
Improved code
Oct 10, 2024
8f070ed
Replaced match_globs with std::regex
Oct 10, 2024
cb7096b
Improved code
Oct 10, 2024
a5a67ca
Fix on regex lang
Oct 10, 2024
ed0c860
possible fix on regex
Oct 10, 2024
974be60
Fixed capio cl regex
Oct 12, 2024
69f16cd
fixes
Oct 14, 2024
5b8bf92
Bugfix on client manager and create handler
Oct 14, 2024
3eaedce
Merge pull request #115 from High-Performance-IO/performance-improv
marcoSanti Oct 14, 2024
9cfc25a
Improved the file_manager_impl.hpp
Oct 14, 2024
dbaba99
Varius fixes and hopefully performance improvements
Oct 14, 2024
eceaf4c
Fixed issue on FSMonitor
Oct 15, 2024
8923920
fix
Oct 15, 2024
d78d41f
fixes
Oct 15, 2024
9c42bd9
Clean code and reduced operation overhead
Oct 16, 2024
0f1c799
Refactor of file-manager component
Oct 16, 2024
e884f4f
fix
Oct 16, 2024
d7aa698
fix
Oct 16, 2024
7c5fb49
fix
Oct 16, 2024
7dff576
fix
Oct 16, 2024
1b1a541
fix
Oct 16, 2024
91a2e78
fix
Oct 16, 2024
fb13db5
fixes
Oct 16, 2024
a8fe195
fixes
Oct 16, 2024
5c75d7b
fix
Oct 16, 2024
36616fc
fix
Oct 16, 2024
7c0fff7
fix
Oct 16, 2024
0d0fd1f
added timestamps to log
Oct 17, 2024
41e5e51
millis_time
Oct 17, 2024
3820c5c
log
Oct 17, 2024
305cbca
log
Oct 17, 2024
8fe23bb
fix
Oct 17, 2024
2a91463
fixes
Oct 17, 2024
d668344
Merge pull request #116 from High-Performance-IO/streaming_fixes
marcoSanti Oct 17, 2024
9edb367
fix
Oct 18, 2024
0d266bf
streaming fix
Oct 18, 2024
dbcbda9
fixes
Oct 18, 2024
57722f2
fix
Oct 18, 2024
4942532
logs
Oct 24, 2024
ed49245
Update CMakeLists.txt
Oct 24, 2024
8435d97
fixes
Oct 25, 2024
9bd965c
logs
Oct 25, 2024
b73428d
Merge pull request #117 from High-Performance-IO/streaming
marcoSanti Oct 25, 2024
f74c4de
Added CI tests on capio-v2 branch
Oct 25, 2024
15d2189
Fixes on consent (#118)
marcoSanti Oct 25, 2024
c2c7657
Capio memory file (#119)
marcoSanti Oct 30, 2024
5deb358
Capio memory file (#119)
marcoSanti Oct 30, 2024
d5d02f3
Added storage service
Oct 30, 2024
614205f
Bugfixes and unit tests
Nov 5, 2024
33c9eb6
Added fetching mechanism for file to store
Nov 19, 2024
5dd1f0f
Ported caches and write requests
Nov 26, 2024
8b2d4a5
Bugfixes
Nov 28, 2024
08f8bf3
Read and write caches
Dec 2, 2024
5feb88d
Merge branch 'capio-v2' into storage-service
marcoSanti Dec 11, 2024
55de1b6
Merge pull request #120 from High-Performance-IO/storage-service
marcoSanti Dec 11, 2024
ceab59b
Added server handler for memory read request
Dec 11, 2024
32ccc1f
Integrated storage service with client manager
Dec 12, 2024
e499a45
Added mechanism to wake waiting threads
Dec 12, 2024
40345b0
Added env var to allways use memory io operations
Dec 12, 2024
6ff1d4b
Integrated caches on libcapio_posix.so
Dec 18, 2024
7881548
Fixed bug in log level
Feb 10, 2025
83ce270
Fixed issue in the initialization of child threads
Feb 10, 2025
0ab0fa8
Merge pull request #123 from High-Performance-IO/log-fix
marcoSanti Feb 10, 2025
98f0b58
Merge pull request #122 from High-Performance-IO/integrate-storage-se…
marcoSanti Feb 24, 2025
0ed80dc
Added MTCL backend to CAPIO
Archabr1el Feb 24, 2025
cc21e7f
Integrated communication backend with CLI args
Archabr1el Feb 24, 2025
56049c3
Began adding multinode tests
Feb 24, 2025
536fa4c
Added backend tests
Feb 25, 2025
2ca8713
Merge pull request #125 from Archabr1el/MTCL
marcoSanti Feb 25, 2025
a052da5
Protected access to unordered_map
Feb 26, 2025
73831f5
Merge pull request #126 from High-Performance-IO/unordered-map-fix
marcoSanti Feb 26, 2025
73f6dda
Added missing syscalls definition aliases
Feb 26, 2025
eef4b1e
Merge pull request #127 from SilenceDesigner/capio-v2
marcoSanti Feb 26, 2025
bf12a09
Refactor and improv of multinode tests
Feb 26, 2025
3e1d60a
Fixed termination bug in fs monitor
Feb 26, 2025
20bf879
Fixed logger consent.hpp
marcoSanti Mar 13, 2025
1f78500
Merge pull request #129 from High-Performance-IO/capio-v2-hostfix
marcoSanti Mar 13, 2025
d22e25a
Hotfix: metadata other than capio_dir folder
Mar 17, 2025
513c88b
Merge pull request #130 from High-Performance-IO/capio-metadata-hotfix
marcoSanti Mar 17, 2025
e209845
Merge pull request #128 from High-Performance-IO/integration-tests-to…
marcoSanti Mar 28, 2025
aca650a
Merge pull request #132 from High-Performance-IO/hotfix
Apr 10, 2025
4a24887
Reduces SHM element allocation
Apr 11, 2025
4efc40f
Merge pull request #133 from High-Performance-IO/Queue-improvements
marcoSanti Apr 18, 2025
16d6d5a
Merge pull request #135 from High-Performance-IO/memory-integration
Apr 9, 2025
ef1e560
Added capio-cl json schema
May 12, 2025
309dc22
Hotfixes to capio
May 27, 2025
6739ad0
Added streaming on directory #138
Jun 4, 2025
91567f7
Capio run (#139)
marcoSanti Jun 22, 2025
149037d
Improved readme.md (#140)
marcoSanti Jun 22, 2025
a1b1d88
Bump version of taywee/args (#142)
marcoSanti Jun 24, 2025
d1f45fc
Bugfixes on directory streaming related code (#143)
marcoSanti Jul 2, 2025
69e49f2
Fixed database and improved resilience of CAPIO (#144)
marcoSanti Jul 5, 2025
4a4942d
Added missing code from dirent.h (#145)
marcoSanti Jul 7, 2025
865b141
Added CapioRemoteFile (#146)
marcoSanti Jul 23, 2025
d0683eb
Improved readdir (#148)
marcoSanti Jul 30, 2025
0acf75b
Added multicast control plane (#147)
marcoSanti Jul 31, 2025
0518cb2
removed CAPIO_DIR from CAPIO_SERVER (#149)
marcoSanti Aug 1, 2025
52dac6f
Metadata hotfix (#151)
marcoSanti Aug 2, 2025
7dad57b
Added Commit-N-Files rule parsing within the CAPIO-CL json parser (#153)
marcoSanti Aug 5, 2025
0b1ee9d
Added SYS_copy_file_range (#154)
marcoSanti Aug 5, 2025
9f25692
Fixed symbolic links within requests (#155)
marcoSanti Aug 7, 2025
a7f0bd2
Hotfix: open handler and capiorun(#156)
marcoSanti Aug 8, 2025
a218d32
Added option to gracefully shut down CAPIO server with SIGUSR1 (#157)
marcoSanti Aug 23, 2025
1d8652c
Update C++ version to C++20 to be fully compatible with MTCL backend …
marcoSanti Aug 23, 2025
399f6ca
Moved global variables into a dedicated class (#159)
marcoSanti Aug 23, 2025
d3f1e1b
Syscall_intercept from alpha-unito (#152)
marcoSanti Aug 28, 2025
52ca55d
HOTFIX: riscv readlinkat fixes (#160)
marcoSanti Aug 28, 2025
dd9269f
Added draft implementation of control plane (#161)
marcoSanti Aug 29, 2025
9af295c
Refactor (#162)
marcoSanti Aug 30, 2025
ee9af5e
Fixed cmake issues (#164)
marcoSanti Sep 11, 2025
09f6ba8
CAPIO-CL exlude section support (#167)
marcoSanti Sep 14, 2025
ba44105
Bugfixes on forbidden paths (#169)
marcoSanti Sep 15, 2025
b758a2f
Backend integration (#163)
marcoSanti Sep 22, 2025
4f7c5a1
Bugfixes on Open - Openat - creat handler (#171)
marcoSanti Sep 24, 2025
e488e38
Fixed integration tests (#172)
marcoSanti Sep 24, 2025
2997b26
Added API server to request shutdown (#166)
marcoSanti Sep 24, 2025
b45603b
Improved performances on cache. Fixed bug on CapioClEngine::isFirable…
marcoSanti Sep 30, 2025
7eed4ec
Improved exclude handling
Oct 22, 2025
110ab22
Logs
Oct 22, 2025
c0a4a49
Logs
Oct 22, 2025
1c7c1c7
Added capio-cl as external project
Nov 26, 2025
bc469d3
Fixed problematic compilation for CAPIO-CL dependency
Dec 15, 2025
7a0d2fe
Fixed tests compilation
Dec 18, 2025
fa94ba6
Fixed servers startup
Dec 18, 2025
c63ffe8
Fix on wfname
Dec 18, 2025
599e2a3
fix
Dec 18, 2025
32cd46f
Check for excluded paths that contains parenthesis
Jan 23, 2026
fed311c
Fixed inlined global variable
Jan 23, 2026
ecc87af
Bugfix on get_workflow_name
Jan 23, 2026
c6fbe8b
Logs
Jan 23, 2026
7ebfcb2
fix
Jan 23, 2026
afc58f8
Added more checks
Jan 23, 2026
ba6749d
Fixes
Jan 23, 2026
ff6038f
Removed useless write_fs handler
Jan 24, 2026
53ce6c7
fixes on FileMonitor
Jan 24, 2026
664e174
bugfix
Jan 25, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 54 additions & 58 deletions .github/workflows/ci-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,20 @@ on:
pull_request:
branches:
- master
- capio-v2
concurrency:
group: build-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
codespell-check:
name: "Check codespell conformance"
runs-on: ubuntu-22.04
continue-on-error: true
steps:
- uses: actions/checkout@v4
- name: "Run codespell"
uses: codespell-project/actions-codespell@v2
with:
exclude_file: capio-tests/unit/server/src/SourceText.hpp
docker-check:
name: "Check Docker image"
runs-on: ubuntu-22.04
Expand All @@ -39,7 +42,9 @@ jobs:
load: true
tags: alphaunito/capio:latest
- name: "Run unit tests with Docker"
working-directory: ${{ github.workspace }}
run: |

echo "Run CAPIO POSIX Unit tests"
docker run --rm \
--env CAPIO_DIR=/tmp \
Expand All @@ -49,64 +54,43 @@ jobs:
capio_posix_unit_tests \
--gtest_break_on_failure \
--gtest_print_time=1

echo "Run CAPIO server Unit tests"
docker run --rm \
--env CAPIO_DIR=/tmp \
--env CAPIO_LOG_LEVEL=-1 \
--name capio-docker \
alphaunito/capio:latest \
capio_server_unit_tests \
--gtest_break_on_failure \
--gtest_print_time=1

echo "Run CAPIO syscall Unit tests"
docker run --rm \
--env CAPIO_DIR=/tmp \
--env CAPIO_LOG_LEVEL=-1 \
--env LD_PRELOAD=libcapio_posix.so \
--name capio-docker \
alphaunito/capio:latest \
capio_server_unit_tests \
--gtest_break_on_failure \
--gtest_print_time=1

echo "Run CAPIO integration tests"
docker run --rm \
--env CAPIO_DIR=/tmp \
--env CAPIO_LOG_LEVEL=-1 \
--env LD_PRELOAD=libcapio_posix.so \
--name capio-docker \
alphaunito/capio:latest \
capio_integration_tests \
--gtest_break_on_failure \
--gtest_print_time=1

- name: "Run backend communication tests"
working-directory: ${{ github.workspace }}
run: docker compose -f capio-tests/multinode/backend/docker-compose.yml up --abort-on-container-exit

- name: "Run integration tests"
working-directory: ${{ github.workspace }}
run: docker compose -f capio-tests/multinode/integration/docker-compose.yml up --abort-on-container-exit


format-check:
name: "Check ${{ matrix.path }} clang-format conformance"
runs-on: ubuntu-22.04
strategy:
matrix:
path:
- "src"
- "tests"
- "capio-posix"
- "capio-server"
- "capio-common"
- "capio-tests"
steps:
- uses: actions/checkout@v4
- name: "Run clang-format style check"
uses: jidicula/clang-format-action@v4.11.0
uses: jidicula/clang-format-action@v4.15.0
with:
clang-format-version: "16"
clang-format-version: "20"
check-path: "${{ matrix.path }}"
unit-tests:
name: "Build ${{ matrix.build_type }} with ${{ matrix.cxx }}"
runs-on: ubuntu-22.04
continue-on-error: true
strategy:
matrix:
build_type:
- Debug
- Release
cxx:
- g++-9
- g++-10
- g++-11
- g++-12
Expand All @@ -126,7 +110,8 @@ jobs:
libopenmpi-dev \
ninja-build \
openmpi-bin \
pkg-config
pkg-config \
cmake
- name: "Get compiler version"
run: |
IFS='-' read -r -a COMPILER <<< "${{ matrix.cxx }}"
Expand Down Expand Up @@ -161,38 +146,48 @@ jobs:
-B ../build \
-S ${GITHUB_WORKSPACE}
cmake --build ../build -j $(nproc)
sudo cmake --install ../build --prefix /usr/local
sudo cmake --install ../build --prefix /usr/local

- name: "Run tests"
id: run-tests
timeout-minutes: 5
timeout-minutes: 2
env:
CAPIO_DIR: ${{ github.workspace }}
CAPIO_LOG_LEVEL: -1
run: |
export LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}"


echo "Run CAPIO syscall Unit tests"
capio_server --no-config &
CAPIO_SERVER_PID=$!
LD_PRELOAD=libcapio_posix.so capio_syscall_unit_tests \
--gtest_break_on_failure --gtest_print_time=1
kill $CAPIO_SERVER_PID
sleep 2


echo "Run CAPIO memory file integration tests"
capio_server --no-config &
CAPIO_SERVER_PID=$!
LD_PRELOAD=libcapio_posix.so capio_memory_file_unit_tests \
--gtest_break_on_failure --gtest_print_time=1
kill $CAPIO_SERVER_PID
sleep 2



echo "Run CAPIO POSIX Unit tests"
capio_posix_unit_tests \
--gtest_break_on_failure \
--gtest_print_time=1

echo "Run CAPIO server Unit tests"
--gtest_print_time=1
sleep 2


echo "Run CAPIO SERVER unit tests"
capio_server_unit_tests \
--gtest_break_on_failure \
--gtest_print_time=1

echo "Run CAPIO syscall Unit tests"
LD_PRELOAD=libcapio_posix.so \
capio_syscall_unit_tests \
--gtest_break_on_failure \
--gtest_print_time=1

echo "Run CAPIO integration tests"
rm -rf /dev/shm/CAPIO*
LD_PRELOAD=libcapio_posix.so \
capio_integration_tests \
--gtest_break_on_failure \
--gtest_print_time=1

- name: "Show client logs on failure"
if: ${{ always() && steps.run-tests.outcome == 'failure' && matrix.build_type == 'Debug' }}
run: tail -v -n +1 capio_logs/posix/$(hostname)/posix_thread_*.log
Expand All @@ -204,6 +199,7 @@ jobs:
run: |
pip install --upgrade gcovr
gcovr \
--gcov-ignore-parse-errors \
--exclude-throw-branches \
--xml coverage.xml \
--gcov-executable "${{ startsWith(matrix.cxx, 'clang-') && format('llvm-cov-{0} gcov', env.CXX_VERSION) || format('gcov-{0}', env.CXX_VERSION) }}" \
Expand Down
62 changes: 0 additions & 62 deletions .github/workflows/release.yml

This file was deleted.

10 changes: 9 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,15 @@ cmake-build-*
files_location*.txt
capio_logs

#Doxygen generated documentation
doxy/html
doxy/latex
doxy/doxygen-awesome-css-*
doxy/theme

# Other
debug
build

.devcontainer
.DS_Store
*.alive_connection
51 changes: 43 additions & 8 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ project(capio
)

# Set required C++ standard
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED TRUE)

# Generate the compile_commands.json file
Expand All @@ -21,6 +21,16 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -pedantic -O0")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")

# Silence warning from G++
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-terminate")
endif ()

# Silence warning from clang
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -pedantic -O0 -Wno-gnu-zero-variadic-macro-arguments -Wno-exceptions")
endif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")

#####################################
# Options
#####################################
Expand All @@ -37,14 +47,21 @@ include(GNUInstallDirs)
#####################################
# Dependencies
#####################################
find_package(MPI REQUIRED)
find_package(Threads REQUIRED)

#####################################
# Definitions
#####################################
add_compile_definitions(CAPIO_VERSION="${CMAKE_PROJECT_VERSION}")

# We check here and define only here the CAPIO_BUILD_TESTS macro
# as it is required by both src/posix and src/server targets
# Later on we check again to add the tests subproject, which cannot
# be added here for issues with include directories
IF (CAPIO_BUILD_TESTS)
add_compile_definitions(CAPIO_BUILD_TESTS)
ENDIF (CAPIO_BUILD_TESTS)

IF (CAPIO_LOG)
IF (CMAKE_BUILD_TYPE STREQUAL "Debug")
message(STATUS "Enabling CAPIO logger")
Expand All @@ -61,8 +78,8 @@ ENDIF (CAPIO_LOG)
#####################################
# Include files and directories
#####################################
file(GLOB_RECURSE CAPIO_COMMON_HEADERS "src/common/capio/*.hpp")
include_directories(src/common)
file(GLOB_RECURSE CAPIO_COMMON_HEADERS "capio-common/capio/*.hpp")
include_directories(${PROJECT_SOURCE_DIR}/capio-common)

IF (CAPIO_LOG AND CMAKE_BUILD_TYPE STREQUAL "Debug")
include_directories("${PROJECT_BINARY_DIR}/include/syscall")
Expand All @@ -71,10 +88,28 @@ ENDIF (CAPIO_LOG AND CMAKE_BUILD_TYPE STREQUAL "Debug")
#####################################
# Targets
#####################################
add_subdirectory(src/posix)
add_subdirectory(src/server)
add_subdirectory(capio-posix)
add_subdirectory(capio-server)

#####################################
# Install capio-run
#####################################
install(
FILES ${PROJECT_SOURCE_DIR}/capio-run/capiorun
DESTINATION ${CMAKE_INSTALL_BINDIR}
PERMISSIONS
OWNER_READ
OWNER_WRITE
OWNER_EXECUTE
GROUP_READ
GROUP_EXECUTE
WORLD_READ
WORLD_EXECUTE
)


IF (CAPIO_BUILD_TESTS)
message(STATUS "Building CAPIO test suite")
add_subdirectory(tests)
ENDIF (CAPIO_BUILD_TESTS)
add_compile_definitions(CAPIO_BUILD_TESTS)
add_subdirectory(capio-tests)
ENDIF (CAPIO_BUILD_TESTS)
Loading