diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 8f9a70a0..459f0d8e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -26,15 +26,15 @@ jobs: with: python-version: ${{ matrix.python-version }} - stress-test-linux: - needs: build-linux - strategy: - fail-fast: false - matrix: - python-version: ["3.12"] - uses: ./.github/workflows/test-linux-wheel.yml - with: - python-version: ${{ matrix.python-version }} - pytest-markers: "-m 'stress_test'" - timeout-minutes: 60 + # stress-test-linux: + # needs: build-linux + # strategy: + # fail-fast: false + # matrix: + # python-version: ["3.12"] + # uses: ./.github/workflows/test-linux-wheel.yml + # with: + # python-version: ${{ matrix.python-version }} + # pytest-markers: "-m 'stress_test'" + # timeout-minutes: 60 diff --git a/.github/workflows/build-mac-wheel.yml b/.github/workflows/build-mac-wheel.yml new file mode 100644 index 00000000..9e354c8d --- /dev/null +++ b/.github/workflows/build-mac-wheel.yml @@ -0,0 +1,80 @@ +name: Build macOS Wheel + +on: + workflow_call: + inputs: + python-version: + required: true + type: string + build-args: + required: false + type: string + default: '' + timeout-minutes: + required: false + type: number + default: 60 + workflow_dispatch: + inputs: + python-version: + description: 'Python version to build' + required: true + type: choice + options: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + default: '3.12' + build-args: + description: 'Additional build arguments (e.g., --config-setting=setup-args=-Denable_sanitizers=true)' + required: false + type: string + default: '' + timeout-minutes: + description: 'Timeout in minutes' + required: false + type: number + default: 60 + +jobs: + build-macos: + runs-on: macos-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Generate meson files + run: | + python3 scripts/generate_meson.py ./src/dbzero/ core + python3 scripts/generate_meson_tests.py tests/ + python3 scripts/generate_meson_dbzero.py dbzero/ + + - name: Configure git + run: | + git config --global user.email "ci@example.com" + git config --global user.name "CI Builder" + rm -f .gitignore + git add . && git commit -m "Update meson files" + + - name: Install build tools + run: pip install build + + - name: Build wheel + run: python3 -m build ${{ inputs.build-args }} + env: + CIBW_SKIP: pp* cp36-* *-musllinux* + CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_ARCHS_LINUX: x86_64 aarch64 + + - uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ inputs.python-version }} + path: dist/*.whl diff --git a/.github/workflows/test-linux-wheel.yml b/.github/workflows/test-linux-wheel.yml index 6762cfc6..f8f2cf3f 100644 --- a/.github/workflows/test-linux-wheel.yml +++ b/.github/workflows/test-linux-wheel.yml @@ -32,17 +32,6 @@ on: - '3.13' - '3.14' default: '3.12' - distribution: - description: 'Linux distribution' - required: true - type: choice - options: - - 'ubuntu' - - 'debian' - - 'arch' - - 'fedora' - - 'opensuse' - default: 'ubuntu' ld-preload: description: 'LD_PRELOAD value (for ASAN: LD_PRELOAD=$(gcc -print-file-name=libasan.so))' required: false diff --git a/meson.build b/meson.build index 2bc9a8df..d4a0a2c3 100644 --- a/meson.build +++ b/meson.build @@ -1,30 +1,76 @@ project('DBZero', 'cpp', default_options :['buildtype=release', 'b_ndebug=if-release']) -if build_machine.system() == 'linux' - add_project_arguments('-Wno-unused-result', +cc = meson.get_compiler('cpp') +compiler_id = cc.get_id() + +message('Compiler ID: ' + compiler_id) +message('Build system: ' + build_machine.system()) + +if build_machine.system() == 'linux' or build_machine.system() == 'darwin' + # Common flags for both GCC and Clang on Linux/Darwin + common_args = [ + '-Wno-unused-result', '-Wno-unused-local-typedefs', '-Wno-deprecated-declarations', - '-Wno-address-of-packed-member', '-Werror=switch', '-Werror=return-type', '-fPIC', - '-no-pie', - '-ftemplate-backtrace-limit=0', '-fno-omit-frame-pointer', '-ldl', '-lm', '-lpthread', - '-fuse-ld=gold', - language : 'cpp') - add_project_arguments('-std=c++17', language : 'cpp') - else + ] + + add_project_arguments(common_args, language : 'cpp') + + # GCC-specific options + if compiler_id == 'gcc' + message('Using GCC-specific options') + gcc_args = [ + '-no-pie', + '-ftemplate-backtrace-limit=0', + '-fuse-ld=gold', + ] + add_project_arguments(gcc_args, language : 'cpp') + endif + + # Clang-specific options + if compiler_id == 'clang' + message('Using Clang-specific options') + clang_args = [ + '-ftemplate-backtrace-limit=0', + # REMOVE BEFORE COMMIT, + '-Wno-inconsistent-missing-override', + '-Wno-unused-private-field', + '-Wno-mismatched-tags', + '-Wno-overloaded-virtual', + '-Wno-class-memaccess' + ] + add_project_arguments(clang_args, language : 'cpp') + # Clang on Linux might support gold linker + if build_machine.system() == 'linux' + add_project_arguments('-fuse-ld=lld', language : 'cpp') + endif + endif + + # Darwin-specific adjustments + if build_machine.system() == 'darwin' + message('Using Darwin-specific options') + # Remove -Wno-address-of-packed-member as it might not be available on all Darwin compilers + else + # Linux-specific: add address-of-packed-member warning suppression + add_project_arguments('-Wno-address-of-packed-member', language : 'cpp') + endif + + add_project_arguments('-std=c++17', language : 'cpp') +else add_project_arguments( '-ldl', '-lm', '-lpthread', language : 'cpp') add_project_arguments('-std:c++20', language : 'cpp') - endif +endif enable_debug_exceptions = get_option('enable_debug_exceptions') @@ -61,16 +107,16 @@ py3_inst = import('python').find_installation('python3', pure: false) install_dir = py3_inst.get_install_dir() / 'dbzero' / 'libs' deps = [] -if build_machine.system() == 'linux' - python_embed_dep = dependency('python3-embed', main:true, required: true) -endif +# if build_machine.system() == 'linux' +# python_embed_dep = dependency('python3-embed', main:true, required: true) +# endif python_deps = py3_inst.dependency() deps += python_deps -if build_machine.system() == 'linux' - deps += python_embed_dep -endif +# if build_machine.system() == 'linux' +# deps += python_embed_dep +# endif all_srcs = [] subdir('src/dbzero') @@ -87,19 +133,21 @@ link_tests_with = link_with tests_sources = [] -gtest_proj = subproject('gtest') -gtest_dep = gtest_proj.get_variable('gtest_dep') -gmock_dep = gtest_proj.get_variable('gmock_dep') -gtest_dep = gtest_proj.get_variable('gtest_dep') -gtest_dep = dependency('gtest', main : true, required : false) -gmock_dep = dependency('gmock', main : true, required : false) - dbzero_lib = static_library('pyzero', [all_srcs], include_directories:include_dirs, dependencies: [deps], install : true, install_dir: install_dir) - +all_deps = [deps] build_tests = get_option('build_tests') if build_tests + gtest_proj = subproject('gtest') + + gtest_dep = gtest_proj.get_variable('gtest_dep') + gmock_dep = gtest_proj.get_variable('gmock_dep') + gtest_dep = gtest_proj.get_variable('gtest_dep') + gtest_dep = dependency('gtest', main : true, required : false) + gmock_dep = dependency('gmock', main : true, required : false) + all_deps += gtest_dep + all_deps += gmock_dep message('Building C++ tests') subdir('tests') e = executable('tests' + build_suffix + '.x', [all_srcs,tests_sources], dependencies: [deps, gtest_dep, gmock_dep], @@ -111,7 +159,7 @@ endif py3avlw = py3_inst.extension_module('dbzero', sources: ['src/dbzero/bindings/python/dbzero.cpp'], - dependencies : [deps, gtest_dep, gmock_dep], + dependencies : all_deps, include_directories:include_dirs, link_with:[dbzero_lib, link_with], link_language : 'cpp', diff --git a/python_tests/test_copy_prefix.py b/python_tests/test_copy_prefix.py index 6cfbf9f1..b2c3b666 100644 --- a/python_tests/test_copy_prefix.py +++ b/python_tests/test_copy_prefix.py @@ -449,4 +449,95 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None): for i in range(copy_id): last_len = validate_copy(i, expected_min_len = last_len) - \ No newline at end of file + + +@pytest.mark.stress_test +def test_copy_prefix_continuous_process_slow_copy(db0_fixture): + px_name = db0.get_current_prefix().name + px_path = os.path.join(DB0_DIR, px_name + ".db0") + db0.set_test_params(sleep_interval = 50) + def validate_current_prefix(expected_len = None, expected_min_len = None): + root = db0.fetch(MemoTestSingleton) + assert not expected_min_len or len(root.value) >= expected_min_len + assert not expected_len or len(root.value) == expected_len + for item in root.value: + assert item.value == "b" * 1024 + return len(root.value) + + def validate_copy(copy_id, expected_len = None, expected_min_len = None): + file_name = f"./test-copy-{copy_id}.db0" + os.remove(px_path) + # restore the copy + os.rename(file_name, px_path) + + print(f"--- Validating copy {copy_id}", flush=True) + db0.init(DB0_DIR, prefix=px_name, read_write=False) + result = validate_current_prefix(expected_len, expected_min_len) + db0.close() + return result + + db0.close() + + # in each 'epoch' we modify prefix while making copies + # then drop the original prefix and restore if from the last copy + epoch_count = 3 + total_len = 0 + for epoch in range(epoch_count): + print(f"=== Epoch {epoch} ===", flush=True) + obj_count = 500 + commit_count = 100 + # start the writer process for a long run + p = multiprocessing.Process(target=writer_process, args=(px_name, obj_count, commit_count, True)) + p.start() + + db0.init(DB0_DIR) + db0.open(px_name, "r") + last_len = 0 + time.sleep(3) + while True: + try: + root = db0.fetch(MemoTestSingleton) + if len(root.value) > 1: + last_len = len(root.value) + break + except Exception as ex: + print("Exception while fetching root:", ex, flush=True) + pass + time.sleep(0.1) + + copy_id = 0 + # copy the prefix multiple times while it is being modified + while True: + if not p.is_alive(): + break + file_name = f"./test-copy-{copy_id}.db0" + if os.path.exists(file_name): + os.remove(file_name) + # copy prefix without opening it, use default step size + print("--- Copying prefix iteration", copy_id, flush=True) + db0.copy_prefix(file_name, prefix=px_name) + print("--- copy finished", flush=True) + copy_id += 1 + if not p.is_alive(): + break + time.sleep(2.5) # wait a bit before next copy + + p.join() + total_len += obj_count * commit_count + + # make final stale copy (i.e. without active modifications) + final_copy = f"./test-copy-final.db0" + if os.path.exists(final_copy): + os.remove(final_copy) + db0.copy_prefix(final_copy, prefix=px_name) + db0.close() + + print("Validating all copies", flush=True) + validate_copy("final", expected_len = total_len) + for i in range(copy_id): + last_len = validate_copy(i, expected_min_len = last_len) + print(f"--- Copy {i} valid with {last_len} objects", flush=True) + # this is the restored version + total_len = last_len + + \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index c52a9e7c..5a966819 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -939,12 +939,12 @@ namespace db0::python { // make sure output is file doesn't point to a directory if (output_file_name.back() == std::filesystem::path::preferred_separator) { - PyErr_Format(PyExc_OSError, "Output file points to a directory: '%s'", output_file_name); + PyErr_Format(PyExc_OSError, "Output file points to a directory: '%s'", output_file_name.c_str()); return nullptr; } // make sure output file does not exist if (db0::CFile::exists(output_file_name)) { - PyErr_Format(PyExc_OSError, "Output file already exists: '%s'", output_file_name); + PyErr_Format(PyExc_OSError, "Output file already exists: '%s'", output_file_name.c_str()); return nullptr; } diff --git a/src/dbzero/bindings/python/shared_py_object.hpp b/src/dbzero/bindings/python/shared_py_object.hpp index 51fa2da0..c942c48e 100644 --- a/src/dbzero/bindings/python/shared_py_object.hpp +++ b/src/dbzero/bindings/python/shared_py_object.hpp @@ -55,7 +55,7 @@ namespace db0::python : m_py_object(other.m_py_object) { static_assert(!ExtRef, "Member only available for non-ExtRef conversion"); - static_assert(other.hasExtRefs, "Source object must have ExtRef"); + //static_assert(other.hasExtRefs, "Source object must have ExtRef"); if (m_py_object) { PyEXT_DECREF(m_py_object); } diff --git a/src/dbzero/core/collections/b_index/bindex_interface.hpp b/src/dbzero/core/collections/b_index/bindex_interface.hpp index e18a41f5..204787cc 100644 --- a/src/dbzero/core/collections/b_index/bindex_interface.hpp +++ b/src/dbzero/core/collections/b_index/bindex_interface.hpp @@ -258,7 +258,7 @@ namespace db0::bindex::interface template struct SizeFunctor > { - static std::size_t execute(const void *this_ptr) { + static std::uint64_t execute(const void *this_ptr) { const db0::v_bindex &index = *reinterpret_cast*>(this_ptr); return index.size(); } @@ -369,7 +369,7 @@ namespace db0::bindex::interface template struct SizeFunctor > { - static size_t execute(const void *) { + static std::uint64_t execute(const void *) { return N; } }; @@ -527,7 +527,7 @@ namespace db0::bindex::interface }; template struct SizeFunctor > { - static size_t execute(const void *this_ptr) { + static std::uint64_t execute(const void *this_ptr) { const db0::v_sorted_vector &index = *reinterpret_cast*>(this_ptr); return index.size(); @@ -683,7 +683,7 @@ namespace db0::bindex::interface template struct SizeFunctor > { - static std::size_t execute(const void *) { + static std::uint64_t execute(const void *) { return 1; } }; @@ -837,7 +837,7 @@ namespace db0::bindex::interface }; template struct SizeFunctor > { - static std::size_t execute(const void *) { + static std::uint64_t execute(const void *) { return 0; } }; diff --git a/src/dbzero/core/collections/full_text/FT_Iterator.cpp b/src/dbzero/core/collections/full_text/FT_Iterator.cpp index c74c42ef..7c005514 100644 --- a/src/dbzero/core/collections/full_text/FT_Iterator.cpp +++ b/src/dbzero/core/collections/full_text/FT_Iterator.cpp @@ -13,12 +13,6 @@ namespace db0 return typeid(key_t); } - template class db0::FT_Iterator; - template class db0::FT_Iterator; - template class db0::FT_Iterator; - template class db0::FT_Iterator>; - template class db0::FT_Iterator>; - template FT_Iterator::FT_Iterator(std::uint64_t uid) : FT_IteratorBase(uid) @@ -97,4 +91,11 @@ namespace db0 return true; } + // Explicit template instantiations - must be after all method definitions + template class db0::FT_Iterator; + template class db0::FT_Iterator; + template class db0::FT_Iterator; + template class db0::FT_Iterator>; + template class db0::FT_Iterator>; + } diff --git a/src/dbzero/core/collections/range_tree/RT_FTIterator.hpp b/src/dbzero/core/collections/range_tree/RT_FTIterator.hpp index 9ec2375f..98004987 100644 --- a/src/dbzero/core/collections/range_tree/RT_FTIterator.hpp +++ b/src/dbzero/core/collections/range_tree/RT_FTIterator.hpp @@ -41,7 +41,7 @@ namespace db0 private: const std::uint64_t m_fixture_uuid = 0; - const Address m_index_addr = 0; + const Address m_index_addr = {}; const RT_Range m_range; std::list > > makeQuery(SharedPtrWrapper tree_ptr, std::optional min, diff --git a/src/dbzero/core/intrusive/intrusive_fwd.hpp b/src/dbzero/core/intrusive/intrusive_fwd.hpp index 8df26d74..40e085f5 100755 --- a/src/dbzero/core/intrusive/intrusive_fwd.hpp +++ b/src/dbzero/core/intrusive/intrusive_fwd.hpp @@ -19,17 +19,6 @@ /// @cond -//std predeclarations -namespace std { - -template -struct equal_to; - -template -struct less; - -} //namespace std{ - //Hash predeclaration template struct hash; diff --git a/src/dbzero/core/memory/Address.hpp b/src/dbzero/core/memory/Address.hpp index 7729a188..9e983eed 100644 --- a/src/dbzero/core/memory/Address.hpp +++ b/src/dbzero/core/memory/Address.hpp @@ -68,13 +68,24 @@ DB0_PACKED_BEGIN return m_value; } - inline AddressType operator+(std::uint64_t offset) const { + inline AddressType operator+(offset_t offset) const { return AddressType(m_value + offset); } - inline AddressType operator-(std::uint64_t offset) const { + inline AddressType operator-(offset_t offset) const { + return AddressType(m_value - offset); + } + + template >> + inline AddressType operator+(std::size_t offset) const { + return AddressType(m_value + offset); + } + + template >> + inline AddressType operator-(std::size_t offset) const { return AddressType(m_value - offset); } + private: store_t m_value = 0; diff --git a/src/dbzero/core/storage/CFile.cpp b/src/dbzero/core/storage/CFile.cpp index 3cb40ef7..6d09701a 100644 --- a/src/dbzero/core/storage/CFile.cpp +++ b/src/dbzero/core/storage/CFile.cpp @@ -51,6 +51,12 @@ namespace db0 auto tp = fs::last_write_time(fs::path(file_name)); auto duration = tp.time_since_epoch(); return std::chrono::duration_cast(duration).count(); + #elif defined(__APPLE__) + struct stat st; + if (stat(file_name, &st)) { + THROWF(db0::IOException) << "CFile::getLastModifiedTime: stat failed"; + }; + return st.st_mtimespec.tv_sec * 1000000000 + st.st_mtimespec.tv_nsec; #else struct stat st; if (stat(file_name, &st)) { diff --git a/src/dbzero/core/storage/Page_IO.cpp b/src/dbzero/core/storage/Page_IO.cpp index 2d468b1f..ba48da2b 100644 --- a/src/dbzero/core/storage/Page_IO.cpp +++ b/src/dbzero/core/storage/Page_IO.cpp @@ -222,7 +222,7 @@ namespace db0 } start_page_num = m_current_page_num; - auto to_read = std::min(max_pages, m_end_page_num - m_current_page_num); + auto to_read = std::min(std::uint64_t(max_pages), m_end_page_num - m_current_page_num); // align with the step size (if defined) if (!!m_step_it) { if (!m_step_it.is_end()) { diff --git a/src/dbzero/object_model/object/lofi_store.hpp b/src/dbzero/object_model/object/lofi_store.hpp index fbb76776..2a5d3889 100644 --- a/src/dbzero/object_model/object/lofi_store.hpp +++ b/src/dbzero/object_model/object/lofi_store.hpp @@ -260,7 +260,7 @@ DB0_PACKED_END template typename lofi_store::const_iterator lofi_store::begin() const { - return { m_data, 0, this->size() }; + return { m_data, 0, static_cast(this->size()) }; } template