diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9b86b2c --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,178 @@ +cmake_minimum_required(VERSION 3.11...3.16) + +project(icf VERSION 1.0 LANGUAGES CXX) +option(PYTHON_SETUP "If invoked by setup.py" OFF) +set(LIBTARGET ${PROJECT_NAME}) +set(PYTARGET ${PROJECT_NAME}_py) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +if(EXISTS $ENV{CONDA_PREFIX}) + set(CMAKE_INSTALL_PREFIX $ENV{CONDA_PREFIX}) + +endif() + + +if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) + # The following settings were copied from + # https://cmake.org/Wiki/CMake_RPATH_handling + # to avoid the rpath issue (issue #11998) that appears on OS X El Capitan + # https://forge.in2p3.fr/issues/11998 + + # use, i.e. don't skip the full RPATH for the build tree + set(CMAKE_SKIP_BUILD_RPATH FALSE) + + # when building, don't use the install RPATH already + # (but later on when installing) + if(PYTHON_SETUP) + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + else() + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + endif() + + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + + # add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +endif() + +find_package(Git QUIET) +# find_package(Python3 COMPONENTS Interpreter) +find_package(pybind11 QUIET) + +set_property(GLOBAL PROPERTY USE_FOLDERS ON) +include(CTest) +include(ExternalProject) + +# Save executables to bin directory +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Dependencies +include(FetchContent) + +if(NOT pybind11_FOUND) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11 + GIT_TAG v2.4.3 + GIT_SHALLOW TRUE + ) + + FetchContent_GetProperties(pybind11) + if(NOT pybind11_POPULATED) + FetchContent_Populate(pybind11) + add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) + endif() +endif() + +if(NOT PYTHON_SETUP) + message(STATUS "${Cyan}doctest${ColourReset}") + FetchContent_Declare( + doctest + GIT_REPOSITORY https://github.com/onqtam/doctest + GIT_TAG 2.3.6 + GIT_SHALLOW TRUE + ) + FetchContent_GetProperties(doctest) + if(NOT doctest_POPULATED) + FetchContent_Populate(doctest) + set(DOCTEST_WITH_TESTS OFF CACHE BOOL "No tests of doctests") + set(DOCTEST_NO_INSTALL ON CACHE BOOL "No doctest install") + add_subdirectory(${doctest_SOURCE_DIR} ${doctest_BINARY_DIR}) + endif() + set(DOCTEST_INCLUDE_DIR ${doctest_SOURCE_DIR}/doctest CACHE INTERNAL "Path to include folder for doctest") +endif() + + +# src +add_library(${LIBTARGET} SHARED src/icfFile.cc) +target_include_directories(${LIBTARGET} PUBLIC + $ + $ + ) +target_compile_features(${LIBTARGET} PUBLIC cxx_std_11) + +if(PYTHON_SETUP AND NOT EXISTS $ENV{CONDA_PREFIX}) + set_target_properties(${LIBTARGET} + PROPERTIES + PREFIX "" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/" + ) +else() + install (TARGETS ${LIBTARGET} EXPORT icf-file-targets LIBRARY DESTINATION lib) + install (DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include) +endif() +# Makes it easier for IDEs to find all headers +source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${HEADER_LIST}) + +#Get git version number +execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=7 + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE VERSION + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) +set(ICF_VERSION_LONG ${VERSION}) +#parse the version information into pieces. +string(REGEX REPLACE "^v([0-9]+)\\..*" "\\1" VERSION_MAJOR "${VERSION}") +string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" VERSION_MINOR "${VERSION}") +string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.([0-9]+).*" "\\1" VERSION_PATCH "${VERSION}") +string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.[0-9]+(.*)" "\\1" VERSION_SHA1 "${VERSION}") +set(ICF_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") +message(STATUS ICF_VERSION) +configure_file( version_config.h.in ${CMAKE_BINARY_DIR}/generated/version_config.h ) +include_directories( ${CMAKE_BINARY_DIR}/generated/) + + +# pybind +# set(PỲTARGET_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/") +# message(${PYTARGET_OUTPUT_DIR}) +pybind11_add_module(${PYTARGET} pybind/module.cc pybind/icfFile.cc) +target_link_libraries(${PYTARGET} PRIVATE ${LIBTARGET}) +set_target_properties(${PYTARGET} + PROPERTIES + PREFIX "" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/" +) +if(NOT PYTHON_SETUP) +# Python stuff +# Creating a symlink to the python package +add_custom_command(TARGET ${PYTARGET} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/icf/" "${CMAKE_CURRENT_BINARY_DIR}/icf/_icf") +# Python stuff +# Creating a symlink to the python tests +add_custom_command(TARGET ${PYTARGET} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/pytest/" "${CMAKE_CURRENT_BINARY_DIR}/pytest") + + +if(Python3_Interpreter_FOUND) + add_test(NAME PythonTests COMMAND python3 -m pytest -r a -v WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pytest") +endif() + + +configure_file(setup.py.in "${PROJECT_BINARY_DIR}/setup.py") + + +# Place the initialization file in the output directory for the Python +# bindings. This will simplify the installation. +CONFIGURE_FILE(__init__.py + ${CMAKE_CURRENT_BINARY_DIR}/icf/__init__.py +) + + +add_executable(test_icf ctests/test_icf.cc)# $) +target_link_libraries(test_icf ${LIBTARGET}) + + +# ctests +add_library(test_main OBJECT ctests/test_main.cc) +target_include_directories(test_main PUBLIC ${DOCTEST_INCLUDE_DIR}) + +add_executable(test_ICFFile ctests/ICFFile.cc $ ) +add_test(NAME test_ICFFile COMMAND test_ICFFile) +target_link_libraries(test_ICFFile ${LIBTARGET}) +target_include_directories(test_ICFFile PUBLIC ctests ${DOCTEST_INCLUDE_DIR}) +target_compile_features(test_ICFFile PRIVATE cxx_std_11) + +endif() \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..9de4b72 --- /dev/null +++ b/__init__.py @@ -0,0 +1,5 @@ +from ._icf import * +from ._ext.icf_py import * +from ._ext.icf_py import _get_version + +__version__ = _get_version() diff --git a/ctests/ICFFile.cc b/ctests/ICFFile.cc new file mode 100644 index 0000000..25193c9 --- /dev/null +++ b/ctests/ICFFile.cc @@ -0,0 +1,50 @@ + +#include "icf/icfFile.h" +#include "doctest.h" +#include +#include +#include +#include + +namespace icf { + + +TEST_CASE("ICFFile") { + icf::ICFFile icf_file(std::string("/tmp/test.icf"),ICFFile::trunc); + std::vector data = {1,2,3,4,2,3,4}; + auto n =3; + for(uint16_t i = 0; idata(); + for(uint16_t i =0; idata(); + for(uint16_t i =0; i + +#include "icf/icfFile.h" +#include + + + +int main(){ + + icf::ICFFile icf_file(std::string("test.icf")); + // std::cout<= pos_start and curr_bunch > 0: + # read bunch trailer bt = self._read_bunch_trailer() @@ -251,6 +258,7 @@ def _scan_sub_file(self, pos_start, pos_end, file_index=0): current_bt_fp -= bt.bunchoff self._file.seek(current_bt_fp) + return rawindex def _construct_file_index(self, rawindex): @@ -267,7 +275,8 @@ def _get_bunch(self, bunch_id): if bunch_id in self._bunch_buffer: return self._bunch_buffer[bunch_id] else: - self._file.seek(self._file_index[bunch_id[0]] +self._bunch_index[bunch_id][0]) + + self._file.seek(self._file_index[bunch_id[0]] + self._bunch_index[bunch_id][0]) # bunch = self._compressor.decompress( bunch = self._file.read( self._bunch_index[bunch_id][1] @@ -315,6 +324,7 @@ def read(self) -> bytes: def __str__(self): s = "{}:\n".format(self.__class__.__name__) + # s += "filename: {}\n".format(self.filename) s += "timestamp: {}\n".format(datetime.fromtimestamp(self.timestamp)) s += "n_entries: {}\n".format(self.n_entries) @@ -322,6 +332,7 @@ def __str__(self): s += "number of sub files: {}\n".format(len(self._file_index)) # for k, v in self.metadata.items(): # s += "{}: {}\n".format(k, v) + s += "file format version: {}".format(self.version) return s diff --git a/include/icf/archive.h b/include/icf/archive.h new file mode 100644 index 0000000..6a2c978 --- /dev/null +++ b/include/icf/archive.h @@ -0,0 +1,381 @@ +/* +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to +*/ + +#ifndef ARCHIVE_H__ +#define ARCHIVE_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +// typedef len_t uint64_t; + +namespace EndianSwapper +{ + class SwapByteBase + { + public: + static bool ShouldSwap() + { + static const uint16_t swapTest = 1; + return !(*((char*)&swapTest) == 1); + } + + static void SwapBytes(uint8_t& v1, uint8_t& v2) + { + uint8_t tmp = v1; + v1 = v2; + v2 = tmp; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + assert(false); // Shoud not be here... + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + if(ShouldSwap()) + return ((uint16_t)v >> 8) | ((uint16_t)v << 8); + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + if(ShouldSwap()) + { + return (SwapByte::Swap((uint32_t)v & 0xffff) << 16) | (SwapByte::Swap(((uint32_t)v & 0xffff0000) >> 16)); + } + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + if(ShouldSwap()) + return (((uint64_t)SwapByte::Swap((uint32_t)(v & 0xffffffffull))) << 32) | (SwapByte::Swap((uint32_t)(v >> 32))); + return v; + } + }; + + template <> + class SwapByte : public SwapByteBase + { + public: + static float Swap(float v) + { + union { float f; uint8_t c[4]; }; + f = v; + if(ShouldSwap()) + { + SwapBytes(c[0], c[3]); + SwapBytes(c[1], c[2]); + } + return f; + } + }; + + template <> + class SwapByte : public SwapByteBase + { + public: + static double Swap(double v) + { + union { double f; uint8_t c[8]; }; + f = v; + if(ShouldSwap()) + { + SwapBytes(c[0], c[7]); + SwapBytes(c[1], c[6]); + SwapBytes(c[2], c[5]); + SwapBytes(c[3], c[4]); + } + return f; + } + }; +} + +template +class Archive +{ + public: + Archive(STREAM_TYPE& stream) : m_stream(stream) + { + } + template + T& getStream(){return m_stream;} + public: + template + const Archive& operator<<(const T& v) const + { + *this & v; + return *this; + } + + template + Archive& operator>>(T& v) + { + *this & v; + return *this; + } + + public: + template + Archive& operator&(T& v) + { + v.Serialize(*this); + return *this; + } + + template + const Archive& operator&(const T& v) const + { + ((T&)v).Serialize(*this); + return *this; + } + + template + Archive& operator&(T (&v)[N]) + { + uint32_t len; + // *this & len; + for(size_t i = 0; i < N; ++i) + *this & v[i]; + return *this; + } + + template + const Archive& operator&(const T (&v)[N]) const + { + uint32_t len = N; + // *this & len; + for(size_t i = 0; i < N; ++i) + *this & v[i]; + return *this; + } + +#define SERIALIZER_FOR_POD(type) \ + Archive& operator&(type& v) \ + { \ + m_stream.read((char*)&v, sizeof(type)); \ + if(!m_stream) { throw std::runtime_error("malformed data"); } \ + v = Swap(v); \ + return *this; \ + } \ + const Archive& operator&(type v) const \ + { \ + v = Swap(v); \ + m_stream.write((const char*)&v, sizeof(type)); \ + return *this; \ + } + + SERIALIZER_FOR_POD(bool) + SERIALIZER_FOR_POD(char) + SERIALIZER_FOR_POD(unsigned char) + SERIALIZER_FOR_POD(short) + SERIALIZER_FOR_POD(unsigned short) + SERIALIZER_FOR_POD(int) + SERIALIZER_FOR_POD(unsigned int) + SERIALIZER_FOR_POD(long) + SERIALIZER_FOR_POD(unsigned long) + SERIALIZER_FOR_POD(long long) + SERIALIZER_FOR_POD(unsigned long long) + SERIALIZER_FOR_POD(float) + SERIALIZER_FOR_POD(double) + + // Archive& operator&(len_t & v) + // { + // char first_byte; + // m_stream.read(&first_byte, sizeof(char)); + // if(!m_stream) { throw std::runtime_error("malformed data"); } + // uint8_t number_of_bytes = (first_byte%4)*2-1; + // m_stream.read((char*)&v, sizeof(number_of_bytes)); + // v+= first_byte; + // v = Swap(v); + // return *this; + // } + + // const Archive& operator&(len_t v) const + // { + // auto vt = 4*v; + // vt = Swap(vt); + + // m_stream.write((const char*)&v, sizeof(len_t )); + // return *this; + // } + + + +#define SERIALIZER_FOR_STL(type) \ + template \ + Archive& operator&(type& v) \ + { \ + uint32_t len; \ + *this & len; \ + for(uint32_t i = 0; i < len; ++i) \ + { \ + T value; \ + *this & value; \ + v.insert(v.end(), value); \ + } \ + return *this; \ + } \ + template \ + const Archive& operator&(const type& v) const \ + { \ + uint32_t len = v.size(); \ + *this & len; \ + for(typename type::const_iterator it = v.begin(); it != v.end(); ++it) \ + *this & *it; \ + return *this; \ + } + +#define SERIALIZER_FOR_STL2(type) \ + template \ + Archive& operator&(type& v) \ + { \ + uint32_t len; \ + *this & len; \ + for(uint32_t i = 0; i < len; ++i) \ + { \ + std::pair value; \ + *this & value; \ + v.insert(v.end(), value); \ + } \ + return *this; \ + } \ + template \ + const Archive& operator&(const type& v) const \ + { \ + uint32_t len = v.size(); \ + *this & len; \ + for(typename type::const_iterator it = v.begin(); it != v.end(); ++it) \ + *this & *it; \ + return *this; \ + } + + SERIALIZER_FOR_STL(std::vector) + SERIALIZER_FOR_STL(std::deque) + SERIALIZER_FOR_STL(std::list) + SERIALIZER_FOR_STL(std::set) + SERIALIZER_FOR_STL(std::multiset) + SERIALIZER_FOR_STL2(std::map) + SERIALIZER_FOR_STL2(std::multimap) + + template + Archive& operator&(std::pair& v) + { + *this & v.first & v.second; + return *this; + } + + template + const Archive& operator&(const std::pair& v) const + { + *this & v.first & v.second; + return *this; + } + + Archive& operator&(std::string& v) + { + uint32_t len; + *this & len; + v.clear(); + char buffer[4096]; + uint32_t toRead = len; + while(toRead != 0) + { + uint32_t l = std::min(toRead, (uint32_t)sizeof(buffer)); + m_stream.read(buffer, l); + if(!m_stream) + throw std::runtime_error("malformed data"); + v += std::string(buffer, l); + toRead -= l; + } + return *this; + } + + const Archive& operator&(const std::string& v) const + { + uint32_t len = v.length(); + *this & len; + m_stream.write(v.c_str(), len); + return *this; + } + + private: + template + T Swap(const T& v) const + { + return EndianSwapper::SwapByte::Swap(v); + } + + private: + STREAM_TYPE& m_stream; +}; + +#endif // ARCHIVE_H__ \ No newline at end of file diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h new file mode 100644 index 0000000..e552ee5 --- /dev/null +++ b/include/icf/icfFile.h @@ -0,0 +1,156 @@ +#ifndef ICF_FILE_H__ +#define ICF_FILE_H__ + +#include "icf/archive.h" +#include +#include +#include +#include +// #include +// #include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + +namespace icf{ + +class ICFFile{ + struct ICFFileHeader{ + + ICFFileHeader():version_(0),compression_(0),unused_(0),ext_head_len_(0){ + strncpy(file_identifier_,"ICF",4); + strncpy(file_sub_identifier_,"",4); + + } + + template + void Serialize(T& archive) + { + time_stamp2_ = std::chrono::system_clock::now(); + time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp2_); + archive & file_identifier_ & + file_sub_identifier_ & + version_ & + compression_ & + time_stamp_& + unused_ & + ext_head_len_; + time_stamp2_ = std::chrono::system_clock::from_time_t(time_stamp_); + + } + + char file_identifier_[4]; + char file_sub_identifier_[4]; + uint16_t version_; + uint16_t compression_; + std::time_t time_stamp_; + std::chrono::time_point time_stamp2_; + uint16_t unused_; + uint16_t ext_head_len_; + }; + + struct ICFBunchTrailer{ + + ICFBunchTrailer(uint64_t file_offset, + uint64_t prev_bunch_offset, + uint64_t bunch_data_offset, + uint32_t n_chunks_in_bunch, + uint32_t bunch_number):file_offset_(file_offset), + prev_bunch_offset_(prev_bunch_offset), + bunch_data_offset_(bunch_data_offset), + n_chunks_in_bunch_(n_chunks_in_bunch), + bunch_number_(bunch_number), + version_(0){ + time_stamp2_ = std::chrono::system_clock::now(); + time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp2_); + } + + template + void Serialize(T& archive) + { + archive & version_; + + + archive & unused_ & + time_stamp_ & + file_offset_ & + prev_bunch_offset_ & + bunch_data_offset_ & + n_chunks_in_bunch_ & + bunch_number_ & + flags_; + + } + + uint16_t version_; + uint16_t unused_; + std::time_t time_stamp_; + std::chrono::time_point time_stamp2_; + uint64_t file_offset_; + uint64_t prev_bunch_offset_; + uint64_t bunch_data_offset_; + uint32_t n_chunks_in_bunch_; + uint32_t bunch_number_; + uint32_t flags_; + + // uint32_t trailer_start_offset_; + + }; + +public: + enum access_mode{read, trunc, append}; + + ICFFile(std::string path, ICFFile::access_mode mode=ICFFile::append, uint32_t bunchsize = 1000000); + + ~ICFFile(); + + void write(const void* data, std::size_t size); + + std::shared_ptr > read_at(uint64_t index); + + void flush(); + + void close(){flush();file_handle_.close(); } + + uint64_t size(){return object_index_.size();} + + std::chrono::time_point get_timestamp(){return file_header_.time_stamp2_;} + + +private: + + void scan_file(uint64_t pos); + + ICFFile::ICFFileHeader file_header_; + std::fstream file_handle_; + uint64_t current_read_pointer_; + uint64_t current_write_pointer_; + std::vector object_index_; + Archive serializer_stream_; + std::vector > > write_buffer_; + uint32_t n_entries_; + uint32_t cbunchoffset_; + uint32_t bunchsize_; + uint64_t last_bunch_fp_; + uint32_t bunch_number_; + // std::vector > read_buffer_; +}; + + + +}//icf namespace + + +#endif \ No newline at end of file diff --git a/pybind/icfFile.cc b/pybind/icfFile.cc new file mode 100644 index 0000000..aafdc02 --- /dev/null +++ b/pybind/icfFile.cc @@ -0,0 +1,36 @@ +// Copyright 2019 Cherenkov Telescope Array Observatory +// This software is distributed under the terms of the BSD-3-Clause license. + +#include "icf/icfFile.h" +#include +#include +#include +#include "version_config.h" +namespace icf { + +namespace py = pybind11; + +py::bytes read_at(ICFFile &obj,uint64_t index){ + auto data = obj.read_at(index); + std::string s(data->begin(),data->end()); + return py::bytes(s); +} + +void write(ICFFile &obj, std::string &data){ + obj.write(data.data(),data.size()); +} + + +void icf_file(py::module &m) { + m.def("_get_version",&getVersion); + + py::class_ icf_file(m, "ICFFile"); + icf_file.def(py::init()); + icf_file.def("read_at",&read_at,"Reads chunk at index", py::arg("index")); + icf_file.def("write",&write,"Writes bytes in chunk at the end of file", py::arg("data")); + icf_file.def("size",&ICFFile::size,"The number of chunks in the file"); + icf_file.def("close",&ICFFile::close,"Closes the file handle"); + icf_file.def("get_timestamp",&ICFFile::get_timestamp,"Returns file creation timestamp"); + +} +} //icf namespace \ No newline at end of file diff --git a/pybind/module.cc b/pybind/module.cc new file mode 100644 index 0000000..a579012 --- /dev/null +++ b/pybind/module.cc @@ -0,0 +1,17 @@ +// Copyright 2019 Cherenkov Telescope Array Observatory +// This software is distributed under the terms of the BSD-3-Clause license. + +#include + +namespace icf { + + +namespace py = pybind11; + +void icf_file(py::module &m); + +PYBIND11_MODULE(icf_py, m) { + icf_file(m); +} + +} // namespace icf diff --git a/pytest/test_icffile.py b/pytest/test_icffile.py index af369af..5b08fd4 100644 --- a/pytest/test_icffile.py +++ b/pytest/test_icffile.py @@ -1,9 +1,9 @@ import pytest from icf import pyicf +from icf import ICFFile import os - -@pytest.fixture(params=[pyicf.ICFFile]) +@pytest.fixture(params=[ICFFile, pyicf.ICFFile]) def icf_impl(request): return request.param @@ -39,6 +39,7 @@ def test_read_while_writing(icf_impl): testdata2 = b'blabasdasdlakaskdlaskd' f.write(testdata1) f.write(testdata2) + assert f.size() == 2, "Correct number of objects in file" assert f.read_at(0) == testdata1, "Read back correct data" assert f.read_at(1) == testdata2, "Read back correct data" @@ -49,6 +50,7 @@ def test_write_read_multiple_bunches(icf_impl): os.remove("/tmp/test.icf") except: pass + f = pyicf.ICFFile("/tmp/test.icf", bunchsize=50) data = b"0" * 26 @@ -81,6 +83,7 @@ def test_read_merged_files(icf_impl): os.system("cat /tmp/test1.icf /tmp/test2.icf >> /tmp/cat.icf") fcat = icf_impl("/tmp/cat.icf") + assert fcat.size() == 2, "Correct number of objects in file" assert fcat.read_at(0) == testdata1, "Correct data from file 1" assert fcat.read_at(1) == testdata2, "Correct data from file 2" @@ -113,6 +116,27 @@ def test_read_merged_files_multiple_bunches(icf_impl): assert fcat.read_at(3) == testdata2, "Correct data from file 2" +def test_extwrie_pyread(): + try: + os.remove("/tmp/ext.icf") + except: + pass + f = ICFFile("/tmp/ext.icf") + testdata1 = b'blablakaskdlaskd' + testdata2 = b'blabasdasdlakaskdlaskd' + f.write(testdata1) + f.write(testdata2) + timestamp = f.get_timestamp() + f.close() + + f_py = pyicf.ICFFile("/tmp/ext.icf") + + assert f_py.get_timestamp() == timestamp, "Consistent time stamp" + assert f_py.size() == 2, "Consistent number of chunks" + assert f_py.read_at(0) == testdata1, "Correct data written in ext and read in py" + assert f_py.read_at(1) == testdata2, "Correct data written in ext and read in py" + + def test_bunch_buffer(): n = 10 bf = pyicf.icffile.BunchBuffer(n) @@ -126,5 +150,3 @@ def test_bunch_buffer(): assert n in bf, "New element in Bunch Buffer" assert 0 not in bf, "Oldest element removed from Bunch Buffer" - - diff --git a/setup.py b/setup.py index c83432a..500d4b2 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,145 @@ -########################################### -#### Do not edit this file ################ -#### It has been auto-generated ########### -########################################### +import os +import pathlib + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as build_ext_orig + + +# class CMakeExtension(Extension): + +# def __init__(self, name): +# # don't invoke the original build_ext for this special extension +# super().__init__(name, sources=[]) + + +# class build_ext(build_ext_orig): + +# def run(self): +# for ext in self.extensions: +# self.build_cmake(ext) +# super().run() + +# def build_cmake(self, ext): +# cwd = pathlib.Path().absolute() + +# # these dirs will be created in build_py, so if you don't have +# # any python sources to bundle, the dirs will be missing +# build_temp = pathlib.Path(self.build_temp) +# build_temp.mkdir(parents=True, exist_ok=True) +# extdir = pathlib.Path(self.get_ext_fullpath(ext.name)) +# extdir.mkdir(parents=True, exist_ok=True) + +# # example of cmake args +# config = 'Debug' if self.debug else 'Release' +# cmake_args = [ +# # '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(extdir.parent.absolute()), +# '-DCMAKE_BUILD_TYPE=' + config, +# '-DPYTARGET_OUTPUT_DIR='+ str(extdir.parent.absolute()), +# '-DPYTHON_SETUP=ON' +# ] + +# # example of build args +# build_args = [ +# '--config', config, +# '--', '-j4', +# ] + +# os.chdir(str(build_temp)) +# self.spawn(['cmake', str(cwd)] + cmake_args) +# if not self.dry_run: +# self.spawn(['cmake', '--build', '.'] + build_args) +# os.chdir(str(cwd)) + +import os +import re +import sys +import sysconfig +import platform +import subprocess +from pathlib import Path + +from distutils.version import LooseVersion +from setuptools import setup, Extension, find_packages +from setuptools.command.build_ext import build_ext +from setuptools.command.test import test as TestCommand + + +class CMakeExtension(Extension): + def __init__(self, name): + Extension.__init__(self, name, sources=[]) + + +class CMakeBuild(build_ext): + def run(self): + try: + out = subprocess.check_output(['cmake', '--version']) + except OSError: + raise RuntimeError( + "CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions)) + env = os.environ.copy() + build_directory = os.path.abspath(self.build_temp) + + cmake_args = [ + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + build_directory, + '-DPYTHON_EXECUTABLE=' + sys.executable, + # '-DCMAKE_INSTALL_PREFIX=' + env['CONDA_PREFIX'], + '-DPYTHON_SETUP=ON' + ] + + cfg = 'Debug' if self.debug else 'Release' + build_args = ['--config', cfg] + + cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + + # Assuming Makefiles + build_args += ['--', '-j2'] + + self.build_args = build_args + + + # env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format( + # env.get('CXXFLAGS', ''), + # self.distribution.get_version()) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + # CMakeLists.txt is in the same directory as this setup.py file + cmake_list_dir = os.path.abspath(os.path.dirname(__file__)) + print('-'*10, 'Running CMake prepare', '-'*40) + subprocess.check_call(['cmake', cmake_list_dir] + cmake_args, + cwd=self.build_temp, env=env) + + print('-'*10, 'Building extensions', '-'*40) + cmake_cmd = ['cmake', '--build', '.'] + self.build_args + subprocess.check_call(cmake_cmd, + cwd=self.build_temp) + + print('-'*10, 'Installing cpp library', '-'*40) + cmake_cmd = ['make', 'install'] + subprocess.check_call(cmake_cmd, + cwd=self.build_temp) + # Move from build temp to final position + for ext in self.extensions: + self.move_output(ext) + + def move_output(self, ext): + build_temp = Path(self.build_temp).resolve() + dest_path = Path(self.get_ext_fullpath(ext.name)).resolve() + source_path = build_temp / self.get_ext_filename(ext.name) + dest_directory = dest_path.parents[0] + dest_directory.mkdir(parents=True, exist_ok=True) + self.copy_file(source_path, dest_path) + + + from setuptools import setup, find_packages + install_requires = ["numpy"] setup( name="pyicf", + version="0.1.0", description="An indexable container file format.", author="Samuel Flis", @@ -16,6 +149,10 @@ provides=["icf"], license="GNU Lesser General Public License v3 or later", install_requires=install_requires, + ext_modules=[CMakeExtension('icf/_ext/icf_py')], + cmdclass={ + 'build_ext': CMakeBuild, + }, classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 3", @@ -29,5 +166,4 @@ # "console_scripts": [ # ] # }, -) - +) \ No newline at end of file diff --git a/setup.py.in b/setup.py.in new file mode 100644 index 0000000..fc1352d --- /dev/null +++ b/setup.py.in @@ -0,0 +1,35 @@ +########################################### +#### Do not edit this file ################ +#### It has been auto-generated ########### +########################################### +from setuptools import setup, find_packages + +setup( + name="pyicf", + + version="@ICF_VERSION@", + description="An indexable container file format.", + author="Samuel Flis", + author_email="samuel.d.flis@gmail.com", + url="https://github.com/sflis/pyicf", + packages=find_packages(), + provides=["icf"], + license="GNU Lesser General Public License v3 or later", + package_data = { + 'icf': ['_ext/*'] + }, + classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + # entry_points={ + # "console_scripts": [ + # ] + # }, +) + diff --git a/src/icfFile.cc b/src/icfFile.cc new file mode 100644 index 0000000..2bfca52 --- /dev/null +++ b/src/icfFile.cc @@ -0,0 +1,120 @@ +#include "icf/icfFile.h" +#include +#include +#include + +using namespace icf; + + + + + +ICFFile::ICFFile(std::string path, ICFFile::access_mode mode, uint32_t bunchsize): + serializer_stream_(file_handle_), + n_entries_(0), + cbunchoffset_(0), + bunchsize_(bunchsize), + last_bunch_fp_(0), + bunch_number_(0){ + auto omode = std::ios_base::in | std::ios_base::binary; + switch(mode){ + case append: + omode |= std::ios_base::app | std::ios_base::out; + break; + case trunc: + omode |= std::ios_base::trunc | std::ios_base::out; + default: + break; + } + + file_handle_.open(path, omode); + + current_read_pointer_ = file_handle_.tellg(); + file_handle_.seekp(0, std::ios_base::end); + current_write_pointer_ = file_handle_.tellp(); + auto length = current_write_pointer_; + file_handle_.seekp(0); + + if(current_write_pointer_>current_read_pointer_ && mode !=trunc){ + serializer_stream_>>file_header_; + scan_file(file_handle_.tellp()); + } + else{ + serializer_stream_<>bt; +} + +ICFFile::~ICFFile(){ + file_handle_.close(); +} + + +void ICFFile::scan_file(uint64_t pos){ + // size_t obj_size; + // auto bheader_size = sizeof(obj_size); + // auto length = current_write_pointer_; + // file_handle_.seekp(pos); + // while(file_handle_.tellp()>obj_size; + // pos += obj_size + bheader_size; + // file_handle_.seekp(pos); + // } +} + +void ICFFile::write(const void* data, std::size_t size){ + auto datacopy = std::make_shared >(std::vector(size)); + memcpy(datacopy->data(),data, size); + write_buffer_.push_back(datacopy); + n_entries_ +=1; + cbunchoffset_ += size; + + if(cbunchoffset_> bunchsize_){ + flush(); + } + +} + +void ICFFile::flush(){ + + if(write_buffer_.size()<1){ + return; + } + + file_handle_.seekp(0, std::ios_base::end); + auto bunch_start_fp = file_handle_.tellp(); + std::vector cbunchindex; + + for(auto &data: write_buffer_){ + cbunchindex.push_back(data->size()); + file_handle_.write(data->data(),data->size()); + } + auto curr_bt_fp = file_handle_.tellp(); + ICFFile::ICFBunchTrailer bunch_trailer(curr_bt_fp, + static_cast(curr_bt_fp) - last_bunch_fp_, + curr_bt_fp - bunch_start_fp, + write_buffer_.size(), + bunch_number_); + serializer_stream_< > ICFFile::read_at(uint64_t index){ + + +} \ No newline at end of file diff --git a/version_config.h.in b/version_config.h.in new file mode 100644 index 0000000..aa855e2 --- /dev/null +++ b/version_config.h.in @@ -0,0 +1,11 @@ +#ifndef ICF_VERSION_CONFIG_H +#define ICF_VERSION_CONFIG_H + +#define ICF_VERSION_LIB @ICF_VERSION@ + +const char* getVersion() +{ + return "@ICF_VERSION@"; +} + +#endif // ICF_VERSION_CONFIG_H \ No newline at end of file