From ae6e72e4b4b91d5485351c7fd6a6a78638e33b28 Mon Sep 17 00:00:00 2001 From: Samuel Flis Date: Mon, 20 Jan 2020 11:05:00 +0100 Subject: [PATCH 01/22] First fragile version of the simple file format with rudimentary pybindings --- CMakeLists.txt | 77 +++++++++ ctests/test_icf.cc | 21 +++ include/icf/archive.h | 356 ++++++++++++++++++++++++++++++++++++++++++ include/icf/icfFile.h | 71 +++++++++ pybind/icfFile.cc | 30 ++++ pybind/module.cc | 17 ++ src/icfFile.cc | 80 ++++++++++ 7 files changed, 652 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 ctests/test_icf.cc create mode 100644 include/icf/archive.h create mode 100644 include/icf/icfFile.h create mode 100644 pybind/icfFile.cc create mode 100644 pybind/module.cc create mode 100644 src/icfFile.cc diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d867710 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,77 @@ +cmake_minimum_required(VERSION 3.11...3.16) + +project(icf VERSION 1.0 LANGUAGES CXX) +set(LIBTARGET ${PROJECT_NAME}) +set(PYTARGET ${PROJECT_NAME}_py) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set_property(GLOBAL PROPERTY USE_FOLDERS ON) + +# Save executables to bin directory +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Dependencies +find_package(pybind11 REQUIRED) +# find_package(Doxygen) + +# if(Doxygen_FOUND) +# add_subdirectory(docs) +# else() +# message(STATUS "Doxygen not found, not building docs") +# endif() + +include(CTest) + +# src +# set(HEADER_LIST include/sstcam/interfaces/WaveformDataPacket.h include/sstcam/interfaces/Waveform.h) +add_library(${LIBTARGET} SHARED src/icfFile.cc) +target_include_directories(${LIBTARGET} PUBLIC + $ + $ + ) +target_compile_features(${LIBTARGET} PUBLIC cxx_std_11) +install (TARGETS ${LIBTARGET} EXPORT icf-file-targets LIBRARY DESTINATION lib) + +# Makes it easier for IDEs to find all headers +source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${HEADER_LIST}) + +# pybind +pybind11_add_module(${PYTARGET} pybind/module.cc pybind/icfFile.cc) +target_link_libraries(${PYTARGET} PRIVATE ${LIBTARGET}) +install(TARGETS ${PYTARGET} LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}) + + +add_executable(test_icf ctests/test_icf.cc)# $) +target_link_libraries(test_icf ${LIBTARGET}) + +# # ctests +# add_library(test_main OBJECT ctests/test_main.cc) + +# add_executable(test_WaveformDataPacket ctests/WaveformDataPacket.cc $) +# add_test(NAME test_WaveformDataPacket COMMAND test_WaveformDataPacket) +# target_link_libraries(test_WaveformDataPacket ${LIBTARGET}) +# target_include_directories(test_WaveformDataPacket PUBLIC ctests) +# target_compile_features(test_WaveformDataPacket PRIVATE cxx_std_11) + +# add_executable(test_Waveform ctests/Waveform.cc $) +# add_test(NAME test_Waveform COMMAND test_Waveform) +# target_link_libraries(test_Waveform ${LIBTARGET}) +# target_include_directories(test_Waveform PUBLIC ctests) +# target_compile_features(test_Waveform PRIVATE cxx_std_11) + +# # data files +# file(GLOB DATA_FILES "${CMAKE_CURRENT_SOURCE_DIR}/share/sstcam/interfaces/*") +# file(COPY ${DATA_FILES} DESTINATION ${CMAKE_BINARY_DIR}/share/sstcam/interfaces/) +# install(DIRECTORY "${CMAKE_BINARY_DIR}/share" DESTINATION .) + +# # Compilation options +# target_compile_options(${LIBTARGET} PUBLIC -O2 -Wall -pedantic -Werror -Wextra) + + +# # Python stuff +# # Creating a symlink to the python package +# add_custom_command(TARGET ${PYTARGET} POST_BUILD +# COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/python/" "${PYTHON_PACKAGE_PATH}/interfaces") +# # Creating a symlink to the python extension in the main python package directory tree +# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "$" "${PYTHON_EXTENSIONS_PATH}/${PYTARGET}") \ No newline at end of file diff --git a/ctests/test_icf.cc b/ctests/test_icf.cc new file mode 100644 index 0000000..0b0fe11 --- /dev/null +++ b/ctests/test_icf.cc @@ -0,0 +1,21 @@ +#include + +#include "icf/icfFile.h" +#include + + + +int main(){ + + icf::ICFFile icf_file(std::string("test.icf")); + std::cout< +*/ + +#ifndef ARCHIVE_H__ +#define ARCHIVE_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace EndianSwapper +{ + class SwapByteBase + { + public: + static bool ShouldSwap() + { + static const uint16_t swapTest = 1; + return (*((char*)&swapTest) == 1); + } + + static void SwapBytes(uint8_t& v1, uint8_t& v2) + { + uint8_t tmp = v1; + v1 = v2; + v2 = tmp; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + assert(false); // Shoud not be here... + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + if(ShouldSwap()) + return ((uint16_t)v >> 8) | ((uint16_t)v << 8); + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + if(ShouldSwap()) + { + return (SwapByte::Swap((uint32_t)v & 0xffff) << 16) | (SwapByte::Swap(((uint32_t)v & 0xffff0000) >> 16)); + } + return v; + } + }; + + template + class SwapByte : public SwapByteBase + { + public: + static T Swap(T v) + { + if(ShouldSwap()) + return (((uint64_t)SwapByte::Swap((uint32_t)(v & 0xffffffffull))) << 32) | (SwapByte::Swap((uint32_t)(v >> 32))); + return v; + } + }; + + template <> + class SwapByte : public SwapByteBase + { + public: + static float Swap(float v) + { + union { float f; uint8_t c[4]; }; + f = v; + if(ShouldSwap()) + { + SwapBytes(c[0], c[3]); + SwapBytes(c[1], c[2]); + } + return f; + } + }; + + template <> + class SwapByte : public SwapByteBase + { + public: + static double Swap(double v) + { + union { double f; uint8_t c[8]; }; + f = v; + if(ShouldSwap()) + { + SwapBytes(c[0], c[7]); + SwapBytes(c[1], c[6]); + SwapBytes(c[2], c[5]); + SwapBytes(c[3], c[4]); + } + return f; + } + }; +} + +template +class Archive +{ + public: + Archive(STREAM_TYPE& stream) : m_stream(stream) + { + } + + public: + template + const Archive& operator<<(const T& v) const + { + *this & v; + return *this; + } + + template + Archive& operator>>(T& v) + { + *this & v; + return *this; + } + + public: + template + Archive& operator&(T& v) + { + v.Serialize(*this); + return *this; + } + + template + const Archive& operator&(const T& v) const + { + ((T&)v).Serialize(*this); + return *this; + } + + template + Archive& operator&(T (&v)[N]) + { + uint32_t len; + // *this & len; + for(size_t i = 0; i < N; ++i) + *this & v[i]; + return *this; + } + + template + const Archive& operator&(const T (&v)[N]) const + { + uint32_t len = N; + // *this & len; + for(size_t i = 0; i < N; ++i) + *this & v[i]; + return *this; + } + +#define SERIALIZER_FOR_POD(type) \ + Archive& operator&(type& v) \ + { \ + m_stream.read((char*)&v, sizeof(type)); \ + if(!m_stream) { throw std::runtime_error("malformed data"); } \ + v = Swap(v); \ + return *this; \ + } \ + const Archive& operator&(type v) const \ + { \ + v = Swap(v); \ + m_stream.write((const char*)&v, sizeof(type)); \ + return *this; \ + } + + SERIALIZER_FOR_POD(bool) + SERIALIZER_FOR_POD(char) + SERIALIZER_FOR_POD(unsigned char) + SERIALIZER_FOR_POD(short) + SERIALIZER_FOR_POD(unsigned short) + SERIALIZER_FOR_POD(int) + SERIALIZER_FOR_POD(unsigned int) + SERIALIZER_FOR_POD(long) + SERIALIZER_FOR_POD(unsigned long) + SERIALIZER_FOR_POD(long long) + SERIALIZER_FOR_POD(unsigned long long) + SERIALIZER_FOR_POD(float) + SERIALIZER_FOR_POD(double) + + +#define SERIALIZER_FOR_STL(type) \ + template \ + Archive& operator&(type& v) \ + { \ + uint32_t len; \ + *this & len; \ + for(uint32_t i = 0; i < len; ++i) \ + { \ + T value; \ + *this & value; \ + v.insert(v.end(), value); \ + } \ + return *this; \ + } \ + template \ + const Archive& operator&(const type& v) const \ + { \ + uint32_t len = v.size(); \ + *this & len; \ + for(typename type::const_iterator it = v.begin(); it != v.end(); ++it) \ + *this & *it; \ + return *this; \ + } + +#define SERIALIZER_FOR_STL2(type) \ + template \ + Archive& operator&(type& v) \ + { \ + uint32_t len; \ + *this & len; \ + for(uint32_t i = 0; i < len; ++i) \ + { \ + std::pair value; \ + *this & value; \ + v.insert(v.end(), value); \ + } \ + return *this; \ + } \ + template \ + const Archive& operator&(const type& v) const \ + { \ + uint32_t len = v.size(); \ + *this & len; \ + for(typename type::const_iterator it = v.begin(); it != v.end(); ++it) \ + *this & *it; \ + return *this; \ + } + + SERIALIZER_FOR_STL(std::vector) + SERIALIZER_FOR_STL(std::deque) + SERIALIZER_FOR_STL(std::list) + SERIALIZER_FOR_STL(std::set) + SERIALIZER_FOR_STL(std::multiset) + SERIALIZER_FOR_STL2(std::map) + SERIALIZER_FOR_STL2(std::multimap) + + template + Archive& operator&(std::pair& v) + { + *this & v.first & v.second; + return *this; + } + + template + const Archive& operator&(const std::pair& v) const + { + *this & v.first & v.second; + return *this; + } + + Archive& operator&(std::string& v) + { + uint32_t len; + *this & len; + v.clear(); + char buffer[4096]; + uint32_t toRead = len; + while(toRead != 0) + { + uint32_t l = std::min(toRead, (uint32_t)sizeof(buffer)); + m_stream.read(buffer, l); + if(!m_stream) + throw std::runtime_error("malformed data"); + v += std::string(buffer, l); + toRead -= l; + } + return *this; + } + + const Archive& operator&(const std::string& v) const + { + uint32_t len = v.length(); + *this & len; + m_stream.write(v.c_str(), len); + return *this; + } + + private: + template + T Swap(const T& v) const + { + return EndianSwapper::SwapByte::Swap(v); + } + + private: + STREAM_TYPE& m_stream; +}; + +#endif // ARCHIVE_H__ \ No newline at end of file diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h new file mode 100644 index 0000000..1f68da0 --- /dev/null +++ b/include/icf/icfFile.h @@ -0,0 +1,71 @@ +#ifndef ICF_FILE_H__ +#define ICF_FILE_H__ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace icf{ + +class ICFFile{ + struct ICFFileHeader{ + + ICFFileHeader(){ + strncpy(file_identifier,"ICF",4); + } + + template + void Serialize(T& archive) + { + archive & file_identifier; + } + + char file_identifier[4]; + }; + +public: + enum access_mode{read, trunc, append}; + + ICFFile(std::string path, ICFFile::access_mode mode=ICFFile::append); + + ~ICFFile(); + + void write(const void* data, std::size_t size); + + std::shared_ptr > read_at(uint64_t index); + + void flush(); + + void close(); + ICFFileHeader file_header_; +private: + + std::fstream file_handle_; + uint64_t current_read_pointer_; + uint64_t current_write_pointer_; + uint64_t data_start_point_; + std::vector object_index_; +}; + + + + +}//icf namespace + + +#endif \ No newline at end of file diff --git a/pybind/icfFile.cc b/pybind/icfFile.cc new file mode 100644 index 0000000..26be224 --- /dev/null +++ b/pybind/icfFile.cc @@ -0,0 +1,30 @@ +// Copyright 2019 Cherenkov Telescope Array Observatory +// This software is distributed under the terms of the BSD-3-Clause license. + +#include "icf/icfFile.h" +#include +#include + +namespace icf { + +namespace py = pybind11; + +py::bytes read_at(ICFFile &obj,uint64_t index){ + auto data = obj.read_at(index); + std::string s(data->begin(),data->end()); + return py::bytes(s); +} + +void write(ICFFile &obj, std::string &data){ + obj.write(data.data(),data.size()); +} + + +void icf_file(py::module &m) { + py::class_ icf_file(m, "ICFFile"); + icf_file.def(py::init()); + icf_file.def("read_at",&read_at); + icf_file.def("write",&write); + +} +} //icf namespace \ No newline at end of file diff --git a/pybind/module.cc b/pybind/module.cc new file mode 100644 index 0000000..a579012 --- /dev/null +++ b/pybind/module.cc @@ -0,0 +1,17 @@ +// Copyright 2019 Cherenkov Telescope Array Observatory +// This software is distributed under the terms of the BSD-3-Clause license. + +#include + +namespace icf { + + +namespace py = pybind11; + +void icf_file(py::module &m); + +PYBIND11_MODULE(icf_py, m) { + icf_file(m); +} + +} // namespace icf diff --git a/src/icfFile.cc b/src/icfFile.cc new file mode 100644 index 0000000..bb843d6 --- /dev/null +++ b/src/icfFile.cc @@ -0,0 +1,80 @@ +#include +#include "icf/archive.h" +#include "icf/icfFile.h" +#include + +using namespace icf; + +ICFFile::ICFFile(std::string path, ICFFile::access_mode mode){ + auto omode = std::ios_base::in | std::ios_base::binary; + switch(mode){ + case append: + omode |= std::ios_base::app | std::ios_base::out; + break; + case trunc: + omode |= std::ios_base::trunc | std::ios_base::out; + default: + break; + } + + file_handle_.open(path,omode); + + current_read_pointer_ = file_handle_.tellg(); + file_handle_.seekp(0, std::ios_base::end); + current_write_pointer_ = file_handle_.tellp(); + auto length = current_write_pointer_; + file_handle_.seekp(0); + // std::cout<current_read_pointer_ && mode !=trunc){ + Archive header_stream(file_handle_); + header_stream>>file_header_; + data_start_point_ = file_handle_.tellp(); + size_t obj_size; + auto bheader_size = sizeof(obj_size); + auto curr_fp = data_start_point_; + + while(file_handle_.tellp()>obj_size; + curr_fp += obj_size + bheader_size; + file_handle_.seekp(curr_fp); + // std::cout< header_stream(file_handle_); + header_stream< serializer_stream(file_handle_); + serializer_stream< > ICFFile::read_at(uint64_t index){ + Archive serializer_stream(file_handle_); + auto fp = object_index_[index]; + size_t obj_size; + file_handle_.seekp(fp); + serializer_stream>>obj_size; + auto data = std::make_shared >(obj_size); + + file_handle_.read((char*) data->data(),obj_size); + return data; +} \ No newline at end of file From b9d02f282d0f9c623b0d595f1c07c29502fc72cb Mon Sep 17 00:00:00 2001 From: Samuel Flis Date: Mon, 20 Jan 2020 11:24:03 +0100 Subject: [PATCH 02/22] Only have one Archive streamer for the for the ICFFile object and update the object_index when writing --- include/icf/icfFile.h | 5 ++++- src/icfFile.cc | 25 +++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h index 1f68da0..aa949cf 100644 --- a/include/icf/icfFile.h +++ b/include/icf/icfFile.h @@ -19,6 +19,7 @@ #include #include +#include "icf/archive.h" namespace icf{ @@ -52,7 +53,8 @@ class ICFFile{ void flush(); void close(); - ICFFileHeader file_header_; + + ICFFileHeader file_header_; private: std::fstream file_handle_; @@ -60,6 +62,7 @@ class ICFFile{ uint64_t current_write_pointer_; uint64_t data_start_point_; std::vector object_index_; + Archive serializer_stream_; }; diff --git a/src/icfFile.cc b/src/icfFile.cc index bb843d6..96489d2 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -1,11 +1,11 @@ -#include -#include "icf/archive.h" #include "icf/icfFile.h" +#include #include + using namespace icf; -ICFFile::ICFFile(std::string path, ICFFile::access_mode mode){ +ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_(file_handle_){ auto omode = std::ios_base::in | std::ios_base::binary; switch(mode){ case append: @@ -27,8 +27,7 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode){ // std::cout<current_read_pointer_ && mode !=trunc){ - Archive header_stream(file_handle_); - header_stream>>file_header_; + serializer_stream_>>file_header_; data_start_point_ = file_handle_.tellp(); size_t obj_size; auto bheader_size = sizeof(obj_size); @@ -36,15 +35,14 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode){ while(file_handle_.tellp()>obj_size; + serializer_stream_>>obj_size; curr_fp += obj_size + bheader_size; file_handle_.seekp(curr_fp); // std::cout< header_stream(file_handle_); - header_stream< serializer_stream(file_handle_); - serializer_stream< > ICFFile::read_at(uint64_t index){ - Archive serializer_stream(file_handle_); auto fp = object_index_[index]; size_t obj_size; file_handle_.seekp(fp); - serializer_stream>>obj_size; + serializer_stream_>>obj_size; auto data = std::make_shared >(obj_size); - file_handle_.read((char*) data->data(),obj_size); + file_handle_.read((char*) data->data(), obj_size); return data; } \ No newline at end of file From acb8a62f580846ab43a687a20eff37ab23f792b9 Mon Sep 17 00:00:00 2001 From: Samuel Flis Date: Wed, 22 Jan 2020 09:40:59 +0100 Subject: [PATCH 03/22] Adding more fields to the file header and writing a small script to test the abilities of the new format --- ctests/test_icf.cc | 2 +- examples/test_cpp_ext.py | 47 ++++++++++++++++++++++++++++++++++++++++ include/icf/icfFile.h | 40 ++++++++++++++++++++++++---------- pybind/icfFile.cc | 2 ++ src/icfFile.cc | 12 +++++----- 5 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 examples/test_cpp_ext.py diff --git a/ctests/test_icf.cc b/ctests/test_icf.cc index 0b0fe11..77d91fa 100644 --- a/ctests/test_icf.cc +++ b/ctests/test_icf.cc @@ -8,7 +8,7 @@ int main(){ icf::ICFFile icf_file(std::string("test.icf")); - std::cout< #include #include #include -#include -#include +// #include +// #include #include #include @@ -19,24 +20,40 @@ #include #include -#include "icf/archive.h" + +#include namespace icf{ class ICFFile{ struct ICFFileHeader{ - ICFFileHeader(){ - strncpy(file_identifier,"ICF",4); + ICFFileHeader():version(0),compression(0),unused(0),ext_head_len(0){ + strncpy(file_identifier_,"ICF",4); + strncpy(file_sub_identifier_,"",4); + } template void Serialize(T& archive) { - archive & file_identifier; + time_stamp_ = std::time(nullptr); + archive & file_identifier_ & + file_sub_identifier_ & + version_ & + compression_ & + time_stamp_& + unused_ & + ext_head_len_; } - char file_identifier[4]; + char file_identifier_[4]; + char file_sub_identifier_[4]; + uint16_t version_; + uint16_t compression_; + std::time_t time_stamp_; + uint16_t unused_; + uint16_t ext_head_len_; }; public: @@ -52,15 +69,16 @@ class ICFFile{ void flush(); - void close(); + void close(){file_handle_.close(); } + + uint64_t size(){return object_index_.size();} - ICFFileHeader file_header_; -private: +private: + ICFFileHeader file_header_; std::fstream file_handle_; uint64_t current_read_pointer_; uint64_t current_write_pointer_; - uint64_t data_start_point_; std::vector object_index_; Archive serializer_stream_; }; diff --git a/pybind/icfFile.cc b/pybind/icfFile.cc index 26be224..43489e4 100644 --- a/pybind/icfFile.cc +++ b/pybind/icfFile.cc @@ -25,6 +25,8 @@ void icf_file(py::module &m) { icf_file.def(py::init()); icf_file.def("read_at",&read_at); icf_file.def("write",&write); + icf_file.def("size",&ICFFile::size); + icf_file.def("close",&ICFFile::close); } } //icf namespace \ No newline at end of file diff --git a/src/icfFile.cc b/src/icfFile.cc index 96489d2..2ca11ba 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -17,21 +17,21 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_ break; } - file_handle_.open(path,omode); + file_handle_.open(path, omode); current_read_pointer_ = file_handle_.tellg(); file_handle_.seekp(0, std::ios_base::end); current_write_pointer_ = file_handle_.tellp(); - auto length = current_write_pointer_; + auto length = current_write_pointer_; file_handle_.seekp(0); // std::cout<current_read_pointer_ && mode !=trunc){ serializer_stream_>>file_header_; - data_start_point_ = file_handle_.tellp(); + current_write_pointer_ = file_handle_.tellp(); size_t obj_size; auto bheader_size = sizeof(obj_size); - auto curr_fp = data_start_point_; + auto curr_fp = current_write_pointer_; while(file_handle_.tellp() Date: Wed, 22 Jan 2020 10:16:06 +0100 Subject: [PATCH 04/22] Added source file for easy env setting. Fixed typo in header. --- CMakeLists.txt | 8 +++++++- env_shell.sh.in | 21 +++++++++++++++++++++ examples/test_cpp_ext.py | 5 ++++- include/icf/icfFile.h | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 env_shell.sh.in diff --git a/CMakeLists.txt b/CMakeLists.txt index d867710..e397ab2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,4 +74,10 @@ target_link_libraries(test_icf ${LIBTARGET}) # add_custom_command(TARGET ${PYTARGET} POST_BUILD # COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/python/" "${PYTHON_PACKAGE_PATH}/interfaces") # # Creating a symlink to the python extension in the main python package directory tree -# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "$" "${PYTHON_EXTENSIONS_PATH}/${PYTARGET}") \ No newline at end of file +# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "$" "${PYTHON_EXTENSIONS_PATH}/${PYTARGET}") + +configure_file( + ${CMAKE_SOURCE_DIR}/env_shell.sh.in + ${CMAKE_BINARY_DIR}/env_shell.sh + @ONLY + ) \ No newline at end of file diff --git a/env_shell.sh.in b/env_shell.sh.in new file mode 100644 index 0000000..b695c02 --- /dev/null +++ b/env_shell.sh.in @@ -0,0 +1,21 @@ +#!/bin/sh +if [ -z "${PYICF_SHELL+x}" ];then + printf "###############################\n" + printf "#Entering py-icf dev shell env#\n" + printf "###############################\n" + + export PYICF_SHELL=1 + # export PATH_BACKUP=${PATH} + export PYTHONPATH=@CMAKE_BINARY_DIR@ + # export PATH=@CMAKE_BINARY_DIR@:@CMAKE_BINARY_DIR@/bin: + export PS1_T_BACKUP=${PS1} + PS1="(py-icf dev)"${PS1} +else + unset PYICF_SHELL + PS1=${PS1_T_BACKUP} + # PATH=${PATH_BACKUP} + # unset PATH_BACKUP + unset PYTHONPATH + unset PS1_T_BACKUP + echo "Exited py-icf dev shell Environment." +fi \ No newline at end of file diff --git a/examples/test_cpp_ext.py b/examples/test_cpp_ext.py index c2e6d9d..5b3d8a1 100644 --- a/examples/test_cpp_ext.py +++ b/examples/test_cpp_ext.py @@ -25,10 +25,11 @@ def read_at(self, ind): # data[int(np.random.uniform(0, 1000 - 1))] = np.random.exponential(2) # frame.add("randomarr", data) -frame["array"] = np.array([2,3,5,6,4,5,.120]) +frame["array"] = np.array([[2,3,5],[6,4,5],[12,3,.120]]) frame["a_list_of_lists"] = [1, 3, 4, 5, [9, 4, 5], (93, 3.034)] print(f.size()) f.write(frame) + print(f.size()) f.close() @@ -36,6 +37,8 @@ def read_at(self, ind): print(f.size()) frame2 = f.read_at(0) print(frame2) +frame2['array'] +print(frame2) frame3 = Frame() frame3['col1'] = np.arange(100) frame3['col2'] = np.random.uniform(0,1,100) diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h index 307c983..fb2cf7e 100644 --- a/include/icf/icfFile.h +++ b/include/icf/icfFile.h @@ -28,7 +28,7 @@ namespace icf{ class ICFFile{ struct ICFFileHeader{ - ICFFileHeader():version(0),compression(0),unused(0),ext_head_len(0){ + ICFFileHeader():version_(0),compression_(0),unused_(0),ext_head_len_(0){ strncpy(file_identifier_,"ICF",4); strncpy(file_sub_identifier_,"",4); From 40f1f615190565f9d57e54d55f880d9b2c2b1468 Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 31 Jan 2020 22:55:17 +0100 Subject: [PATCH 05/22] Working on project structure for sensible development and deployment --- CMakeLists.txt | 57 ++++++++++++++++++++++++++++++---------- __init__.py | 5 ++++ env_shell.sh.in | 2 +- examples/test_cpp_ext.py | 26 ++++++++++++++++-- icf/__init__.py | 19 +++----------- icf/frame.py | 4 ++- pybind/icfFile.cc | 10 ++++--- setup.py | 24 ++++++++++------- version_config.h.in | 13 +++++++++ 9 files changed, 113 insertions(+), 47 deletions(-) create mode 100644 __init__.py create mode 100644 version_config.h.in diff --git a/CMakeLists.txt b/CMakeLists.txt index e397ab2..9c8ec40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,8 @@ set(PYTARGET ${PROJECT_NAME}_py) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) + +find_package(Git QUIET) set_property(GLOBAL PROPERTY USE_FOLDERS ON) # Save executables to bin directory @@ -13,18 +15,10 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) # Dependencies find_package(pybind11 REQUIRED) -# find_package(Doxygen) - -# if(Doxygen_FOUND) -# add_subdirectory(docs) -# else() -# message(STATUS "Doxygen not found, not building docs") -# endif() include(CTest) # src -# set(HEADER_LIST include/sstcam/interfaces/WaveformDataPacket.h include/sstcam/interfaces/Waveform.h) add_library(${LIBTARGET} SHARED src/icfFile.cc) target_include_directories(${LIBTARGET} PUBLIC $ @@ -36,10 +30,50 @@ install (TARGETS ${LIBTARGET} EXPORT icf-file-targets LIBRARY DESTINATION lib) # Makes it easier for IDEs to find all headers source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${HEADER_LIST}) +#Get git version number +execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=7 + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE ICF_VERSION_LIB + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) +configure_file( version_config.h.in ${CMAKE_BINARY_DIR}/generated/version_config.h ) +include_directories( ${CMAKE_BINARY_DIR}/generated/) + + # pybind pybind11_add_module(${PYTARGET} pybind/module.cc pybind/icfFile.cc) target_link_libraries(${PYTARGET} PRIVATE ${LIBTARGET}) -install(TARGETS ${PYTARGET} LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}) +set_target_properties(${PYTARGET} + PROPERTIES + PREFIX "" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/" +) + +# Python stuff +# Creating a symlink to the python package +add_custom_command(TARGET ${PYTARGET} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/icf/" "${CMAKE_CURRENT_BINARY_DIR}/icf/_icf") +# Creating a symlink to the python extension in the main python package directory tree +# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext") +# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "$" "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/${PYTARGET}") + + +configure_file( + ${CMAKE_SOURCE_DIR}/env_shell.sh.in + ${CMAKE_BINARY_DIR}/env_shell.sh + @ONLY + ) + +# Place the initialization file in the output directory for the Python +# bindings. This will simplify the installation. +CONFIGURE_FILE(__init__.py + ${CMAKE_CURRENT_BINARY_DIR}/icf/__init__.py +) + +# Ditto for the setup file. +CONFIGURE_FILE(setup.py + ${CMAKE_CURRENT_BINARY_DIR}/setup.py +) add_executable(test_icf ctests/test_icf.cc)# $) @@ -76,8 +110,3 @@ target_link_libraries(test_icf ${LIBTARGET}) # # Creating a symlink to the python extension in the main python package directory tree # add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "$" "${PYTHON_EXTENSIONS_PATH}/${PYTARGET}") -configure_file( - ${CMAKE_SOURCE_DIR}/env_shell.sh.in - ${CMAKE_BINARY_DIR}/env_shell.sh - @ONLY - ) \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..f90fc51 --- /dev/null +++ b/__init__.py @@ -0,0 +1,5 @@ + +from ._icf import * +from ._ext.icf_py import * +from ._ext.icf_py import _get_version +__version__ = _get_version() \ No newline at end of file diff --git a/env_shell.sh.in b/env_shell.sh.in index b695c02..d1ad943 100644 --- a/env_shell.sh.in +++ b/env_shell.sh.in @@ -6,7 +6,7 @@ if [ -z "${PYICF_SHELL+x}" ];then export PYICF_SHELL=1 # export PATH_BACKUP=${PATH} - export PYTHONPATH=@CMAKE_BINARY_DIR@ + export PYTHONPATH=@CMAKE_BINARY_DIR@/icf # export PATH=@CMAKE_BINARY_DIR@:@CMAKE_BINARY_DIR@/bin: export PS1_T_BACKUP=${PS1} PS1="(py-icf dev)"${PS1} diff --git a/examples/test_cpp_ext.py b/examples/test_cpp_ext.py index 5b3d8a1..0519ed2 100644 --- a/examples/test_cpp_ext.py +++ b/examples/test_cpp_ext.py @@ -43,8 +43,30 @@ def read_at(self, ind): frame3['col1'] = np.arange(100) frame3['col2'] = np.random.uniform(0,1,100) frame4 = Frame.deserialize(frame3.serialize()) -print(frame4) +# print(frame4) import pandas as pd df = pd.DataFrame(dict(frame4)) -print(df) +# print(df) + +frame = Frame() +#frame["array"] = np.array([[2,3,5],[6,4,5],[12,3,.120]]) +frame['large array'] = np.random.uniform(0,1000,(600,10,10)) +frame["a_list_of_lists"] = [1, 3, 4, 5, [9, 4, 5], (93, 3.034)] +data = frame.serialize() +print(len(data)) +# f = icf_py.ICFFile('TestingSpeed.icf') +f = FrameFile('TestingSpeed.icf') +for i in range(100000): + # f.write(bytes(data)) + f.write(frame) +f.close() + + + + + + + + + diff --git a/icf/__init__.py b/icf/__init__.py index 9665ef7..ae614dd 100644 --- a/icf/__init__.py +++ b/icf/__init__.py @@ -1,17 +1,4 @@ -from . import version +from icf._icf import io, frame -__version__ = version.get_version(pep440=False) - -from .io import IndexedContainerWriter, IndexedContainerReader - - -# FrameFileHeader: -# def __init__(self): - - -# class FrameWriter(IndexedContainerWriter): -# def __init__(self, filename: str, **kwargs): -# super().__init__(filename, header_ext=CHECFileHeader(3).pack(), **kwargs) - -# def write(self, frame): -# super().write(frame.serialize()) +from .frame import Frame +from .io import IndexedContainerWriter, IndexedContainerReader \ No newline at end of file diff --git a/icf/frame.py b/icf/frame.py index b32476e..ff17bf2 100644 --- a/icf/frame.py +++ b/icf/frame.py @@ -4,7 +4,6 @@ import logging import sys - class SerializationDispatcher: """Summary @@ -364,3 +363,6 @@ def __str__(self): def __repr__(self): return self.__str__() + + +__all__ = [Frame] diff --git a/pybind/icfFile.cc b/pybind/icfFile.cc index 43489e4..85a4a07 100644 --- a/pybind/icfFile.cc +++ b/pybind/icfFile.cc @@ -4,7 +4,7 @@ #include "icf/icfFile.h" #include #include - +#include "version_config.h" namespace icf { namespace py = pybind11; @@ -21,11 +21,13 @@ void write(ICFFile &obj, std::string &data){ void icf_file(py::module &m) { + m.def("_get_version",&getVersion); + py::class_ icf_file(m, "ICFFile"); icf_file.def(py::init()); - icf_file.def("read_at",&read_at); - icf_file.def("write",&write); - icf_file.def("size",&ICFFile::size); + icf_file.def("read_at",&read_at,"Reads chunk at index", py::arg("index")); + icf_file.def("write",&write,"Writes bytes in chunk at the end of file", py::arg("data")); + icf_file.def("size",&ICFFile::size,"The number of chunks in the file"); icf_file.def("close",&ICFFile::close); } diff --git a/setup.py b/setup.py index 19c1c8f..aa54fa5 100644 --- a/setup.py +++ b/setup.py @@ -5,18 +5,18 @@ install_requires = ["numpy", "pyparsing", "pyyaml", "protobuf"] # -from shutil import copyfile, rmtree +# from shutil import copyfile, rmtree -if not os.path.exists("tmps"): - os.makedirs("tmps") -copyfile("icf/version.py", "tmps/version.py") -__import__("tmps.version") -package = sys.modules["tmps"] -package.version.update_release_version("icf") +# if not os.path.exists("tmps"): +# os.makedirs("tmps") +# copyfile("icf/version.py", "tmps/version.py") +# __import__("tmps.version") +# package = sys.modules["tmps"] +# package.version.update_release_version("icf") setup( name="pyicf", - version=package.version.get_version(pep440=True), + version="",#,package.version.get_version(pep440=True), description="An indexable container file format.", author="Samuel Flis", author_email="samuel.d.flis@gmail.com", @@ -28,6 +28,12 @@ extras_requires={ #'encryption': ['cryptography'] }, + # package_dir = { + # '': '${CMAKE_CURRENT_BINARY_DIR}' + # }, + package_data = { + '': ['icf_py.cpython-37m-x86_64-linux-gnu.so'] + }, classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 3", @@ -44,4 +50,4 @@ ) -rmtree("tmps") +# rmtree("tmps") diff --git a/version_config.h.in b/version_config.h.in new file mode 100644 index 0000000..0235cb0 --- /dev/null +++ b/version_config.h.in @@ -0,0 +1,13 @@ +#ifndef ICF_VERSION_CONFIG_H +#define ICF_VERSION_CONFIG_H + +// define your version_libinterface +#define ICF_VERSION_LIB @ICF_VERSION_LIB@ + +// alternatively you could add your global method getLibInterfaceVersion here +const char* getVersion() +{ + return "@ICF_VERSION_LIB@"; +} + +#endif // ICF_VERSION_CONFIG_H \ No newline at end of file From c1b5027d36d1268439400c310ab7d6223347f1ca Mon Sep 17 00:00:00 2001 From: Samuel Date: Sat, 1 Feb 2020 00:36:22 +0100 Subject: [PATCH 06/22] Generating the setup.py with cmake magic using a two level template which is executed with a configure file in a cmake script while building --- CMakeLists.txt | 25 ++++++++++++++++++------- render-variables.cmake.in | 1 + setup.py => setup.py.in.in | 16 +++------------- 3 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 render-variables.cmake.in rename setup.py => setup.py.in.in (71%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c8ec40..002879f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,9 +53,24 @@ set_target_properties(${PYTARGET} # Creating a symlink to the python package add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/icf/" "${CMAKE_CURRENT_BINARY_DIR}/icf/_icf") -# Creating a symlink to the python extension in the main python package directory tree -# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext") -# add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "$" "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/${PYTARGET}") +# Generating the correct setup.py file +file(GENERATE OUTPUT setup.py.in INPUT setup.py.in.in) +configure_file(render-variables.cmake.in render-variables.cmake @ONLY) +add_custom_command( + OUTPUT "${PROJECT_BINARY_DIR}/setup.py" + COMMAND "${CMAKE_COMMAND}" + -DPROJECT_BINARY_DIR="${PROJECT_BINARY_DIR}" + -DPROJECT_NAME="${PROJECT_NAME}" + -DICF_VERSION_LIB="${ICF_VERSION_LIB}" + + -P "${PROJECT_BINARY_DIR}/render-variables.cmake" + MAIN_DEPENDENCY "${PROJECT_BINARY_DIR}/setup.py.in" + DEPENDS "${PROJECT_BINARY_DIR}/render-variables.cmake" +) +add_custom_target( + make-setupy.py ALL + DEPENDS "${PROJECT_BINARY_DIR}/setup.py" +) configure_file( @@ -70,10 +85,6 @@ CONFIGURE_FILE(__init__.py ${CMAKE_CURRENT_BINARY_DIR}/icf/__init__.py ) -# Ditto for the setup file. -CONFIGURE_FILE(setup.py - ${CMAKE_CURRENT_BINARY_DIR}/setup.py -) add_executable(test_icf ctests/test_icf.cc)# $) diff --git a/render-variables.cmake.in b/render-variables.cmake.in new file mode 100644 index 0000000..2e9b17a --- /dev/null +++ b/render-variables.cmake.in @@ -0,0 +1 @@ +configure_file("${PROJECT_BINARY_DIR}/setup.py.in" "${PROJECT_BINARY_DIR}/setup.py" @ONLY) \ No newline at end of file diff --git a/setup.py b/setup.py.in.in similarity index 71% rename from setup.py rename to setup.py.in.in index aa54fa5..3d6d758 100644 --- a/setup.py +++ b/setup.py.in.in @@ -2,21 +2,11 @@ import os import sys -install_requires = ["numpy", "pyparsing", "pyyaml", "protobuf"] - -# -# from shutil import copyfile, rmtree - -# if not os.path.exists("tmps"): -# os.makedirs("tmps") -# copyfile("icf/version.py", "tmps/version.py") -# __import__("tmps.version") -# package = sys.modules["tmps"] -# package.version.update_release_version("icf") +install_requires = ["numpy"] setup( name="pyicf", - version="",#,package.version.get_version(pep440=True), + version="@ICF_VERSION_LIB@", description="An indexable container file format.", author="Samuel Flis", author_email="samuel.d.flis@gmail.com", @@ -32,7 +22,7 @@ # '': '${CMAKE_CURRENT_BINARY_DIR}' # }, package_data = { - '': ['icf_py.cpython-37m-x86_64-linux-gnu.so'] + 'icf': ['_ext/$'] }, classifiers=[ "Programming Language :: Python", From 52e3c05a8331489e85f3733fb4cfa71c344f4e49 Mon Sep 17 00:00:00 2001 From: Samuel Date: Sat, 1 Feb 2020 23:16:59 +0100 Subject: [PATCH 07/22] Added exceptions for the read_at method. Timestamps are now created using the std::chrono library and a get_timestamp method was added --- include/icf/icfFile.h | 10 +++++++--- pybind/icfFile.cc | 4 +++- setup.py.in.in | 16 ++++++---------- src/icfFile.cc | 6 ++---- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h index fb2cf7e..2ec70cd 100644 --- a/include/icf/icfFile.h +++ b/include/icf/icfFile.h @@ -22,7 +22,7 @@ #include #include - +#include namespace icf{ class ICFFile{ @@ -37,7 +37,8 @@ class ICFFile{ template void Serialize(T& archive) { - time_stamp_ = std::time(nullptr); + time_stamp_2 = std::chrono::system_clock::now(); + time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp_2); archive & file_identifier_ & file_sub_identifier_ & version_ & @@ -45,6 +46,8 @@ class ICFFile{ time_stamp_& unused_ & ext_head_len_; + time_stamp_2 = std::chrono::system_clock::from_time_t(time_stamp_); + } char file_identifier_[4]; @@ -52,6 +55,7 @@ class ICFFile{ uint16_t version_; uint16_t compression_; std::time_t time_stamp_; + std::chrono::time_point time_stamp_2; uint16_t unused_; uint16_t ext_head_len_; }; @@ -73,7 +77,7 @@ class ICFFile{ uint64_t size(){return object_index_.size();} - + std::chrono::time_point get_timestamp(){return file_header_.time_stamp_2;} private: ICFFileHeader file_header_; std::fstream file_handle_; diff --git a/pybind/icfFile.cc b/pybind/icfFile.cc index 85a4a07..aafdc02 100644 --- a/pybind/icfFile.cc +++ b/pybind/icfFile.cc @@ -3,6 +3,7 @@ #include "icf/icfFile.h" #include +#include #include #include "version_config.h" namespace icf { @@ -28,7 +29,8 @@ void icf_file(py::module &m) { icf_file.def("read_at",&read_at,"Reads chunk at index", py::arg("index")); icf_file.def("write",&write,"Writes bytes in chunk at the end of file", py::arg("data")); icf_file.def("size",&ICFFile::size,"The number of chunks in the file"); - icf_file.def("close",&ICFFile::close); + icf_file.def("close",&ICFFile::close,"Closes the file handle"); + icf_file.def("get_timestamp",&ICFFile::get_timestamp,"Returns file creation timestamp"); } } //icf namespace \ No newline at end of file diff --git a/setup.py.in.in b/setup.py.in.in index 3d6d758..b4a677e 100644 --- a/setup.py.in.in +++ b/setup.py.in.in @@ -1,3 +1,7 @@ +########################################### +#### Do not edit this file ################ +#### It has been auto-generated ########### +########################################### from setuptools import setup, find_packages import os import sys @@ -10,18 +14,12 @@ setup( description="An indexable container file format.", author="Samuel Flis", author_email="samuel.d.flis@gmail.com", - url="https://github.com/sflis/SSDAQ", + url="https://github.com/sflis/pyicf", packages=find_packages(), provides=["icf"], license="GNU Lesser General Public License v3 or later", install_requires=install_requires, - extras_requires={ - #'encryption': ['cryptography'] - }, - # package_dir = { - # '': '${CMAKE_CURRENT_BINARY_DIR}' - # }, - package_data = { + package_data = { 'icf': ['_ext/$'] }, classifiers=[ @@ -39,5 +37,3 @@ setup( # }, ) - -# rmtree("tmps") diff --git a/src/icfFile.cc b/src/icfFile.cc index 2ca11ba..e03229a 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -24,7 +24,6 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_ current_write_pointer_ = file_handle_.tellp(); auto length = current_write_pointer_; file_handle_.seekp(0); - // std::cout<current_read_pointer_ && mode !=trunc){ serializer_stream_>>file_header_; @@ -38,7 +37,6 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_ serializer_stream_>>obj_size; curr_fp += obj_size + bheader_size; file_handle_.seekp(curr_fp); - // std::cout< > ICFFile::read_at(uint64_t index){ - auto fp = object_index_[index]; + + auto fp = object_index_.at(index); size_t obj_size; file_handle_.seekp(fp); serializer_stream_>>obj_size; From 87f94b1f7c440f966940f705a0795d8b5da21ba4 Mon Sep 17 00:00:00 2001 From: Samuel Date: Mon, 10 Feb 2020 12:12:11 +0100 Subject: [PATCH 08/22] Update, restore point --- CMakeLists.txt | 33 +++++++++++++++- __init__.py | 1 - ctests/ICFFile.cc | 50 +++++++++++++++++++++++++ ctests/test_main.cc | 5 +++ include/icf/archive.h | 3 +- include/icf/icfFile.h | 87 ++++++++++++++++++++++++++++++++++++++++--- src/icfFile.cc | 79 +++++++++++++++++++++++++++++++++------ 7 files changed, 238 insertions(+), 20 deletions(-) create mode 100644 ctests/ICFFile.cc create mode 100644 ctests/test_main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 002879f..2c9f622 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,25 @@ find_package(pybind11 REQUIRED) include(CTest) +include(ExternalProject) +ExternalProject_Add( + doctest + PREFIX ${CMAKE_BINARY_DIR}/doctest + GIT_REPOSITORY https://github.com/onqtam/doctest.git + TIMEOUT 10 + UPDATE_COMMAND ${GIT_EXECUTABLE} pull + GIT_SHALLOW 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD ON +) + +# Expose required variable (DOCTEST_INCLUDE_DIR) to parent scope +ExternalProject_Get_Property(doctest source_dir) +set(DOCTEST_INCLUDE_DIR ${source_dir}/doctest CACHE INTERNAL "Path to include folder for doctest") + + # src add_library(${LIBTARGET} SHARED src/icfFile.cc) target_include_directories(${LIBTARGET} PUBLIC @@ -86,13 +105,25 @@ CONFIGURE_FILE(__init__.py ) - add_executable(test_icf ctests/test_icf.cc)# $) target_link_libraries(test_icf ${LIBTARGET}) + + # # ctests +# ctests +add_library(test_main OBJECT ctests/test_main.cc) +target_include_directories(test_main PUBLIC ${DOCTEST_INCLUDE_DIR}) +add_executable(test_ICFFile ctests/ICFFile.cc $ ) +add_test(NAME test_ICFFile COMMAND test_ICFFile) +target_link_libraries(test_ICFFile ${LIBTARGET}) +target_include_directories(test_ICFFile PUBLIC ctests ${DOCTEST_INCLUDE_DIR}) +target_compile_features(test_ICFFile PRIVATE cxx_std_11) + # add_library(test_main OBJECT ctests/test_main.cc) + + # add_executable(test_WaveformDataPacket ctests/WaveformDataPacket.cc $) # add_test(NAME test_WaveformDataPacket COMMAND test_WaveformDataPacket) # target_link_libraries(test_WaveformDataPacket ${LIBTARGET}) diff --git a/__init__.py b/__init__.py index f90fc51..84183ef 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,3 @@ - from ._icf import * from ._ext.icf_py import * from ._ext.icf_py import _get_version diff --git a/ctests/ICFFile.cc b/ctests/ICFFile.cc new file mode 100644 index 0000000..25193c9 --- /dev/null +++ b/ctests/ICFFile.cc @@ -0,0 +1,50 @@ + +#include "icf/icfFile.h" +#include "doctest.h" +#include +#include +#include +#include + +namespace icf { + + +TEST_CASE("ICFFile") { + icf::ICFFile icf_file(std::string("/tmp/test.icf"),ICFFile::trunc); + std::vector data = {1,2,3,4,2,3,4}; + auto n =3; + for(uint16_t i = 0; idata(); + for(uint16_t i =0; idata(); + for(uint16_t i =0; i + T& getStream(){return m_stream;} public: template const Archive& operator<<(const T& v) const diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h index 2ec70cd..ebf783f 100644 --- a/include/icf/icfFile.h +++ b/include/icf/icfFile.h @@ -26,6 +26,7 @@ namespace icf{ class ICFFile{ +public: struct ICFFileHeader{ ICFFileHeader():version_(0),compression_(0),unused_(0),ext_head_len_(0){ @@ -37,8 +38,8 @@ class ICFFile{ template void Serialize(T& archive) { - time_stamp_2 = std::chrono::system_clock::now(); - time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp_2); + time_stamp2_ = std::chrono::system_clock::now(); + time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp2_); archive & file_identifier_ & file_sub_identifier_ & version_ & @@ -46,7 +47,7 @@ class ICFFile{ time_stamp_& unused_ & ext_head_len_; - time_stamp_2 = std::chrono::system_clock::from_time_t(time_stamp_); + time_stamp2_ = std::chrono::system_clock::from_time_t(time_stamp_); } @@ -55,12 +56,12 @@ class ICFFile{ uint16_t version_; uint16_t compression_; std::time_t time_stamp_; - std::chrono::time_point time_stamp_2; + std::chrono::time_point time_stamp2_; uint16_t unused_; uint16_t ext_head_len_; }; -public: + enum access_mode{read, trunc, append}; ICFFile(std::string path, ICFFile::access_mode mode=ICFFile::append); @@ -77,8 +78,11 @@ class ICFFile{ uint64_t size(){return object_index_.size();} - std::chrono::time_point get_timestamp(){return file_header_.time_stamp_2;} + std::chrono::time_point get_timestamp(){return file_header_.time_stamp2_;} private: + + void scan_file(uint64_t pos); + ICFFileHeader file_header_; std::fstream file_handle_; uint64_t current_read_pointer_; @@ -90,6 +94,77 @@ class ICFFile{ +class ICFFileV2{ + struct ICFBunchTrailer{ + + ICFBunchTrailer():version_(0){ + time_stamp2_ = std::chrono::system_clock::now(); + time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp2_); + } + + void Serialize(Archive& archive) + { + archive & version_; + + + archive & time_stamp_ & + file_offset_ & + prev_bunch_offset_ & + bunch_data_offset_ & + n_chunks_in_bunch_ & + bunch_number_ & + flags_; + + } + + uint16_t version_; + std::time_t time_stamp_; + std::chrono::time_point time_stamp2_; + uint64_t file_offset_; + uint64_t prev_bunch_offset_; + uint64_t bunch_data_offset_; + uint32_t n_chunks_in_bunch_; + uint32_t bunch_number_; + uint32_t flags_; + + uint32_t trailer_start_offset_; + + }; + +public: + enum access_mode{read, trunc, append}; + + ICFFileV2(std::string path, ICFFileV2::access_mode mode=ICFFileV2::append); + + ~ICFFileV2(); + + void write(const void* data, std::size_t size); + + std::shared_ptr > read_at(uint64_t index); + + void flush(); + + void close(){file_handle_.close(); } + + uint64_t size(){return object_index_.size();} + + std::chrono::time_point get_timestamp(){return file_header_.time_stamp2_;} +private: + + void scan_file(uint64_t pos); + + ICFFile::ICFFileHeader file_header_; + std::fstream file_handle_; + uint64_t current_read_pointer_; + uint64_t current_write_pointer_; + std::vector object_index_; + Archive serializer_stream_; + std::vector > write_buffer_; + // std::vector > read_buffer_; +}; + + + }//icf namespace diff --git a/src/icfFile.cc b/src/icfFile.cc index e03229a..560a482 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -27,17 +27,7 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_ if(current_write_pointer_>current_read_pointer_ && mode !=trunc){ serializer_stream_>>file_header_; - current_write_pointer_ = file_handle_.tellp(); - size_t obj_size; - auto bheader_size = sizeof(obj_size); - auto curr_fp = current_write_pointer_; - - while(file_handle_.tellp()>obj_size; - curr_fp += obj_size + bheader_size; - file_handle_.seekp(curr_fp); - } + scan_file(file_handle_.tellp()); } else{ serializer_stream_< > ICFFile::read_at(uint64_t index){ file_handle_.read((char*) data->data(), obj_size); return data; +} + + +void ICFFile::scan_file(uint64_t pos){ + size_t obj_size; + auto bheader_size = sizeof(obj_size); + auto length = current_write_pointer_; + file_handle_.seekp(pos); + while(file_handle_.tellp()>obj_size; + pos += obj_size + bheader_size; + file_handle_.seekp(pos); + } +} + + + + +ICFFileV2::ICFFileV2(std::string path, ICFFileV2::access_mode mode):serializer_stream_(file_handle_){ + auto omode = std::ios_base::in | std::ios_base::binary; + switch(mode){ + case append: + omode |= std::ios_base::app | std::ios_base::out; + break; + case trunc: + omode |= std::ios_base::trunc | std::ios_base::out; + default: + break; + } + + file_handle_.open(path, omode); + + current_read_pointer_ = file_handle_.tellg(); + file_handle_.seekp(0, std::ios_base::end); + current_write_pointer_ = file_handle_.tellp(); + auto length = current_write_pointer_; + file_handle_.seekp(0); + + if(current_write_pointer_>current_read_pointer_ && mode !=trunc){ + serializer_stream_>>file_header_; + scan_file(file_handle_.tellp()); + } + else{ + serializer_stream_<>bt; +} + +ICFFileV2::~ICFFileV2(){ + file_handle_.close(); +} + + +void ICFFileV2::scan_file(uint64_t pos){ + size_t obj_size; + auto bheader_size = sizeof(obj_size); + auto length = current_write_pointer_; + file_handle_.seekp(pos); + while(file_handle_.tellp()>obj_size; + pos += obj_size + bheader_size; + file_handle_.seekp(pos); + } } \ No newline at end of file From 87541408b3526beb6da385db3d408e85a56de5fc Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 13 Feb 2020 11:32:48 +0100 Subject: [PATCH 09/22] Restore point dev --- CMakeLists.txt | 12 ++ __init__.py | 3 +- examples/test_cpp_ext.py | 34 ++--- icf/__init__.py | 2 +- icf/frame.py | 2 + icf/icffile.py | 267 +++++++++++++++++++++++++++++++++++++++ icf/io.py | 22 +++- icf/version.py | 184 --------------------------- include/icf/archive.h | 2 +- include/icf/icfFile.h | 9 +- pytest/test_icffile.py | 18 +++ src/icfFile.cc | 4 +- 12 files changed, 339 insertions(+), 220 deletions(-) create mode 100644 icf/icffile.py delete mode 100644 icf/version.py create mode 100644 pytest/test_icffile.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c9f622..cb9fd74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) find_package(Git QUIET) +find_package(Python3 COMPONENTS Interpreter) + set_property(GLOBAL PROPERTY USE_FOLDERS ON) # Save executables to bin directory @@ -72,6 +74,16 @@ set_target_properties(${PYTARGET} # Creating a symlink to the python package add_custom_command(TARGET ${PYTARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/icf/" "${CMAKE_CURRENT_BINARY_DIR}/icf/_icf") +# Python stuff +# Creating a symlink to the python tests +add_custom_command(TARGET ${PYTARGET} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/pytest/" "${CMAKE_CURRENT_BINARY_DIR}/pytest") + + +if(Python3_Interpreter_FOUND) + add_test(NAME PythonTests COMMAND Python3::Interpreter -m pytest -r a -v WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pytest") +endif() + # Generating the correct setup.py file file(GENERATE OUTPUT setup.py.in INPUT setup.py.in.in) configure_file(render-variables.cmake.in render-variables.cmake @ONLY) diff --git a/__init__.py b/__init__.py index 84183ef..9de4b72 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,5 @@ from ._icf import * from ._ext.icf_py import * from ._ext.icf_py import _get_version -__version__ = _get_version() \ No newline at end of file + +__version__ = _get_version() diff --git a/examples/test_cpp_ext.py b/examples/test_cpp_ext.py index 0519ed2..344c217 100644 --- a/examples/test_cpp_ext.py +++ b/examples/test_cpp_ext.py @@ -2,6 +2,8 @@ from icf import frame from icf.frame import Frame import numpy as np + + class FrameFile(icf_py.ICFFile): def __init__(self, path): super().__init__(path) @@ -13,10 +15,7 @@ def read_at(self, ind): return Frame.deserialize(super().read_at(ind)) - - - -f = FrameFile('Testing_python_ext.icf') +f = FrameFile("Testing_python_ext.icf") frame = Frame() frame.add("rawarray", np.arange(100)) @@ -25,7 +24,7 @@ def read_at(self, ind): # data[int(np.random.uniform(0, 1000 - 1))] = np.random.exponential(2) # frame.add("randomarr", data) -frame["array"] = np.array([[2,3,5],[6,4,5],[12,3,.120]]) +frame["array"] = np.array([[2, 3, 5], [6, 4, 5], [12, 3, 0.120]]) frame["a_list_of_lists"] = [1, 3, 4, 5, [9, 4, 5], (93, 3.034)] print(f.size()) f.write(frame) @@ -33,40 +32,31 @@ def read_at(self, ind): print(f.size()) f.close() -f = FrameFile('Testing_python_ext.icf') +f = FrameFile("Testing_python_ext.icf") print(f.size()) frame2 = f.read_at(0) print(frame2) -frame2['array'] +frame2["array"] print(frame2) frame3 = Frame() -frame3['col1'] = np.arange(100) -frame3['col2'] = np.random.uniform(0,1,100) +frame3["col1"] = np.arange(100) +frame3["col2"] = np.random.uniform(0, 1, 100) frame4 = Frame.deserialize(frame3.serialize()) # print(frame4) import pandas as pd + df = pd.DataFrame(dict(frame4)) # print(df) frame = Frame() -#frame["array"] = np.array([[2,3,5],[6,4,5],[12,3,.120]]) -frame['large array'] = np.random.uniform(0,1000,(600,10,10)) +# frame["array"] = np.array([[2,3,5],[6,4,5],[12,3,.120]]) +frame["large array"] = np.random.uniform(0, 1000, (600, 10, 10)) frame["a_list_of_lists"] = [1, 3, 4, 5, [9, 4, 5], (93, 3.034)] data = frame.serialize() print(len(data)) # f = icf_py.ICFFile('TestingSpeed.icf') -f = FrameFile('TestingSpeed.icf') +f = FrameFile("TestingSpeed.icf") for i in range(100000): # f.write(bytes(data)) f.write(frame) f.close() - - - - - - - - - - diff --git a/icf/__init__.py b/icf/__init__.py index ae614dd..467619a 100644 --- a/icf/__init__.py +++ b/icf/__init__.py @@ -1,4 +1,4 @@ from icf._icf import io, frame from .frame import Frame -from .io import IndexedContainerWriter, IndexedContainerReader \ No newline at end of file +from .io import IndexedContainerWriter, IndexedContainerReader diff --git a/icf/frame.py b/icf/frame.py index ff17bf2..de71376 100644 --- a/icf/frame.py +++ b/icf/frame.py @@ -4,6 +4,7 @@ import logging import sys + class SerializationDispatcher: """Summary @@ -137,6 +138,7 @@ def deserialize(cls, data): class C(SerializationDispatcher, types=[complex]): encode = struct.Struct(" 0 and ("a" in omode or "r" in omode): + self._file.seek(0) + ( + fd, + fd_ext, + self.version, + self.compression, + self.timestamp, + _, + ext_len, + ) = self._file_header.unpack(self._file.read(self._file_header.size)) + self.file_identifier_ext = fd_ext + self.header_ext = self._file.read(ext_len) + print(self.filesize) + raw_index = self._scan_file(self._file.tell(), self.filesize) + self._construct_file_index(raw_index) + + else: + self.timestamp = int(datetime.now().timestamp()) + header_ext = b"" if header_ext is None else header_ext + self._file.write( + self._file_header.pack( + "ICF".encode(), + self.file_identifier_ext.encode(), + 0, + self.compression, + self.timestamp, + 0, + len(header_ext), + ) + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._writer.close() + + def get_timestamp(self): + return datetime.fromtimestamp(self.timestamp) + + def write(self, data: bytes): + """ Writes a stream of bytes to file + + args: + data (bytes): bytes to be writen to file + """ + + self._write_buffer.append(data) + self._cbunchoffset += len(data) + # self._cbunchindex.append(len(data)) + self.n_entries += 1 + if self._cbunchoffset > self.bunchsize: + self.flush() + + def _write(self, data: bytes): + self._file.write(data) + + def flush(self): + """Flushes any data in buffer to file. + + """ + if len(self._write_buffer) < 1: + return + self._file.seek(0, os.SEEK_END) + + bunch_start_fp = self._file.tell() # self._fp + # writing the data bunch + cbunchindex = [] + bytes_buff = bytearray() + for data in self._write_buffer: + bytes_buff.extend(data) + cbunchindex.append(len(data)) + self._write(bytes_buff) + curr_bt_fp = self._file.tell() + # Constructing and writing bunch trailer header + bunch_index_trailer = self._bunch_trailer_header.pack( + self.version, + 0, + int(datetime.now().timestamp()), + curr_bt_fp, + curr_bt_fp - self._last_bunch_fp, + curr_bt_fp - bunch_start_fp, + len(self._write_buffer), + self._bunch_number, + 0, + ) + self._write(bunch_index_trailer) + + # constructing the index and writing it in the bunch trailer + n = len(self._write_buffer) + bunch_index = struct.pack("<{}I".format(n), *cbunchindex) + self._write(bunch_index) + + # Write offset to begining of bunch trailer + self._write(struct.pack("I", self._file.tell() - curr_bt_fp)) + + # Keep the file pointer for the current bunch + self._last_bunch_fp = curr_bt_fp + + self._file.flush() + # reseting/updating the last bunch descriptors + self._write_buffer.clear() + self._cbunchoffset = 0 + self._bunch_number += 1 + + def close(self): + self.flush() + self._file.close() + + def _scan_file(self, pos_start, pos_end): + # We find the last bunch trailer by reading the last 4 bytes which + # encodes the offset to said bunch trailer + self._file.seek(pos_end - 4) + bt_start_offset = struct.unpack("= pos_start and curr_bunch > 0: + print(self._file.tell(), pos_start) + # read bunch trailer + last_bunch_trailer = self._file.read(self._bunch_trailer_header.size) + ( + version, + _, + timestamp, + fileoff, + bunchoff, + dataoff, + ndata, + bunch_n, + flags + ) = self._bunch_trailer_header.unpack(last_bunch_trailer) + curr_bunch = bunch_n + index = struct.unpack("{}I".format(ndata), self._file.read(ndata * 4)) + objsizes = np.array(index, dtype=np.uint32) + rawindex[(0, bunch_n)] = self._BunchTrailer( + bunchoff, # Offset to earlier bunch or file header if first bunch + dataoff, # Offset to beginning of data in bunch + fileoff, # Offset to beginning of file + dataoff, # Size of data bunch + ndata, # number of objects in bunch + [0] + list(np.cumsum(objsizes[:-1])), # Object offsets in bunch + objsizes, # object sizes + ) + self._file.seek(self._file.tell() - bunchoff) + print('Ndata',ndata,self._file.tell(),pos_start,curr_bunch) + return rawindex + + def _construct_file_index(self, rawindex): + + for k, bunch in sorted(rawindex.items()): + self._rawindex[k] = bunch + self._bunch_index[k] = (bunch.fileoff - bunch.dataoff, bunch.bunchsize) + for i, obj in enumerate(bunch.index): + self._index.append((k, int(obj), int(bunch.objsize[i]))) + self.n_entries = len(self._index) + + + def _get_bunch(self, bunch_id): + + if bunch_id in self._bunch_buffer: + return self._bunch_buffer[bunch_id] + else: + self._file.seek(self._bunch_index[bunch_id][0]) + # bunch = self._compressor.decompress( + bunch = self._file.read( + self._file_index[bunch_id[0]] + self._bunch_index[bunch_id][1] + ) + # ) + self._bunch_buffer[bunch_id] = bunch + return bunch + + def read_at(self, ind: int) -> bytes: + """Reads one object at the index indicated by `ind` + + Args: + ind (int): the index of the object to be read + + Returns: + bytes: that represent the object + + Raises: + IndexError: if index out of range + """ + + if ind > self.n_entries - 1: + raise IndexError( + "The requested file object ({}) is out of range".format(ind) + ) + obji = self._index[ind] + if True: # self._compressed: + bunch = self._get_bunch(obji[0]) + return bunch[obji[1]: obji[1] + obji[2]] + # else: + # fpos = self.file_index[obji[0][0]] + self._bunch_index[obji[0]][0] + obji[1] + # self.file.seek(fpos) + + # return self.file.read(obji[2]) + + +from collections import deque + + +class BunchBuffer(dict): + def __init__(self, size): + self.size = size + self.queue = deque() + + def __setitem__(self, key, obj): + self.queue.append((key, obj)) + super().__setitem__(key, obj) + if len(self.queue) > self.size: + bunch = self.queue.popleft() + super().pop(bunch[0]) \ No newline at end of file diff --git a/icf/io.py b/icf/io.py index 3d62769..003d9aa 100644 --- a/icf/io.py +++ b/icf/io.py @@ -443,9 +443,14 @@ def __init__(self, file): self.file = file self.file.seek(0) fileheader = self.file.read(fileheader_def.size) - marker, extmarker, self._version, self._timestamp, self._compressed, self._lenheadext = fileheader_def.unpack( - fileheader - ) + ( + marker, + extmarker, + self._version, + self._timestamp, + self._compressed, + self._lenheadext, + ) = fileheader_def.unpack(fileheader) if marker[:3].decode() != "SOF": raise TypeError("This file appears not to be a stream object file (SOF)") if self._version != self._protocol_v: @@ -487,9 +492,14 @@ def _scan_file(self): while self.file.tell() > self._fp_start: # read bunch trailer last_bunch_trailer = self.file.read(self._bunch_trailer_header.size) - bunchoff, dataoff, fileoff, crc, ndata, bunch_n = self._bunch_trailer_header.unpack( - last_bunch_trailer - ) + ( + bunchoff, + dataoff, + fileoff, + crc, + ndata, + bunch_n, + ) = self._bunch_trailer_header.unpack(last_bunch_trailer) # read bunch index self.file.seek( diff --git a/icf/version.py b/icf/version.py deleted file mode 100644 index 3c75693..0000000 --- a/icf/version.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Get version identification from git. - - -The update_release_version() function writes the current version to the -VERSION file. This function should be called before packaging a release version. - -Use the get_version() function to get the version string, including the latest -commit, from git. -If git is not available the VERSION file will be read. - -Heres an example of such a version string: - - v0.2.0.post58+git57440dc - - -This code was taken from here: -https://github.com/cta-observatory/ctapipe/blob/master/ctapipe/version.py -which in turn based it on: -https://github.com/aebrahim/python-git-version - -Combining ideas from -http://blogs.nopcode.org/brainstorm/2013/05/20/pragmatic-python-versioning-via-setuptools-and-git-tags/ -and Python Versioneer -https://github.com/warner/python-versioneer -but being much more lightwheight - -""" -from __future__ import print_function -from subprocess import check_output, CalledProcessError -from os import path, name, devnull, environ, listdir - -__all__ = ("get_version",) - -CURRENT_DIRECTORY = path.dirname(path.abspath(__file__)) -VFILE = "_version_cache.py" -VERSION_FILE = path.join(CURRENT_DIRECTORY, VFILE) - -GIT_COMMAND = "git" - - -import sys - - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -if name == "nt": - - def find_git_on_windows(): - """find the path to the git executable on windows""" - # first see if git is in the path - try: - check_output(["where", "/Q", "git"]) - # if this command succeeded, git is in the path - return "git" - # catch the exception thrown if git was not found - except CalledProcessError: - pass - # There are several locations git.exe may be hiding - possible_locations = [] - # look in program files for msysgit - if "PROGRAMFILES(X86)" in environ: - possible_locations.append( - "%s/Git/cmd/git.exe" % environ["PROGRAMFILES(X86)"] - ) - if "PROGRAMFILES" in environ: - possible_locations.append("%s/Git/cmd/git.exe" % environ["PROGRAMFILES"]) - # look for the github version of git - if "LOCALAPPDATA" in environ: - github_dir = "%s/GitHub" % environ["LOCALAPPDATA"] - if path.isdir(github_dir): - for subdir in listdir(github_dir): - if not subdir.startswith("PortableGit"): - continue - possible_locations.append( - "%s/%s/bin/git.exe" % (github_dir, subdir) - ) - for possible_location in possible_locations: - if path.isfile(possible_location): - return possible_location - # git was not found - return "git" - - GIT_COMMAND = find_git_on_windows() - - -def get_git_describe_version(abbrev=7): - """return the string output of git desribe""" - try: - with open(devnull, "w") as fnull: - arguments = [GIT_COMMAND, "describe", "--tags", "--abbrev=%d" % abbrev] - return ( - check_output(arguments, cwd=CURRENT_DIRECTORY, stderr=fnull) - .decode("ascii") - .strip() - ) - except (OSError, CalledProcessError): - return None - - -def format_git_describe(git_str, pep440=False): - """format the result of calling 'git describe' as a python version""" - - if "-" not in git_str: # currently at a tag - formatted_str = git_str - else: - # formatted as version-N-githash - # want to convert to version.postN-githash - git_str = git_str.replace("-", ".post", 1) - if pep440: # does not allow git hash afterwards - formatted_str = git_str.split("-")[0] - else: - formatted_str = git_str.replace("-g", "+git") - - # need to remove the "v" to have a proper python version - if formatted_str.startswith("v"): - formatted_str = formatted_str[1:] - - return formatted_str - - -def read_release_version(): - """Read version information from VERSION file""" - try: - from ._version_cache import version - - if len(version) == 0: - version = None - return version - except ImportError: - return "unknown" - - -def update_release_version(fpath, pep440=False): - """Release versions are stored in a file called VERSION. - This method updates the version stored in the file. - This function should be called when creating new releases. - It is called by setup.py when building a package. - - - pep440: bool - When True, this function returns a version string suitable for - a release as defined by PEP 440. When False, the githash (if - available) will be appended to the version string. - - """ - version = get_version(pep440=pep440) - with open(path.join(fpath, VFILE), "w") as outfile: - outfile.write("version='{}'".format(version)) - outfile.write("\n") - - -def get_version(pep440=False): - """Tracks the version number. - - pep440: bool - When True, this function returns a version string suitable for - a release as defined by PEP 440. When False, the githash (if - available) will be appended to the version string. - - The file VERSION holds the version information. If this is not a git - repository, then it is reasonable to assume that the version is not - being incremented and the version returned will be the release version as - read from the file. - - However, if the script is located within an active git repository, - git-describe is used to get the version information. - - The file VERSION will need to be changed manually. - """ - - raw_git_version = get_git_describe_version() - if not raw_git_version: # not a git repository - return read_release_version() - - git_version = format_git_describe(raw_git_version, pep440=pep440) - - return git_version - - -if __name__ == "__main__": - print(get_version()) diff --git a/include/icf/archive.h b/include/icf/archive.h index efdc364..b530751 100644 --- a/include/icf/archive.h +++ b/include/icf/archive.h @@ -50,7 +50,7 @@ namespace EndianSwapper static bool ShouldSwap() { static const uint16_t swapTest = 1; - return (*((char*)&swapTest) == 1); + return !(*((char*)&swapTest) == 1); } static void SwapBytes(uint8_t& v1, uint8_t& v2) diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h index ebf783f..95b8caf 100644 --- a/include/icf/icfFile.h +++ b/include/icf/icfFile.h @@ -23,6 +23,7 @@ #include #include + namespace icf{ class ICFFile{ @@ -97,7 +98,7 @@ class ICFFile{ class ICFFileV2{ struct ICFBunchTrailer{ - ICFBunchTrailer():version_(0){ + ICFBunchTrailer(std::weak_ptr >):version_(0){ time_stamp2_ = std::chrono::system_clock::now(); time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp2_); } @@ -107,7 +108,8 @@ class ICFFileV2{ archive & version_; - archive & time_stamp_ & + archive & unused_ & + time_stamp_ & file_offset_ & prev_bunch_offset_ & bunch_data_offset_ & @@ -116,8 +118,9 @@ class ICFFileV2{ flags_; } - + uint16_t version_; + uint16_t unused_; std::time_t time_stamp_; std::chrono::time_point time_stamp2_; uint64_t file_offset_; diff --git a/pytest/test_icffile.py b/pytest/test_icffile.py new file mode 100644 index 0000000..e6e3c92 --- /dev/null +++ b/pytest/test_icffile.py @@ -0,0 +1,18 @@ +from icf._icf import icffile + + +def test_write_and_readback_icffile(): + f = icffile.ICFFile("/tmp/test.icf",'trunc') + testdata1 = b'blablakaskdlaskd' + testdata2 = b'blabasdasdlakaskdlaskd' + f.write(testdata1) + f.write(testdata2) + original_timestamp = f.get_timestamp() + assert f.n_entries == 2, "Correct number of entries" + f.close() + # open file again + f = icffile.ICFFile("/tmp/test.icf") + assert f.get_timestamp() == original_timestamp, "Read back correct timestamp" + assert f.n_entries == 2, "Read back correct number of entries" + assert f.read_at(0) == testdata1, "Read back correct data" + assert f.read_at(1) == testdata2, "Read back correct data" diff --git a/src/icfFile.cc b/src/icfFile.cc index 560a482..8909f33 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -109,8 +109,8 @@ ICFFileV2::ICFFileV2(std::string path, ICFFileV2::access_mode mode):serializer_s serializer_stream_<>bt; + // auto bt = ICFBunchTrailer(); + // serializer_stream_>>bt; } ICFFileV2::~ICFFileV2(){ From b945261905b753c58bd7515e93bead3cfcfa53df Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 13 Feb 2020 22:27:19 +0100 Subject: [PATCH 10/22] update --- icf/__init__.py | 4 +- icf/io.py | 666 ------------------------------------- icf/pyicf/__init__.py | 2 + icf/{ => pyicf}/icffile.py | 41 ++- pytest/test_icffile.py | 22 +- 5 files changed, 62 insertions(+), 673 deletions(-) delete mode 100644 icf/io.py create mode 100644 icf/pyicf/__init__.py rename icf/{ => pyicf}/icffile.py (85%) diff --git a/icf/__init__.py b/icf/__init__.py index 467619a..6b0f7ec 100644 --- a/icf/__init__.py +++ b/icf/__init__.py @@ -1,4 +1,4 @@ -from icf._icf import io, frame +from icf._icf import frame, pyicf from .frame import Frame -from .io import IndexedContainerWriter, IndexedContainerReader + diff --git a/icf/io.py b/icf/io.py deleted file mode 100644 index 003d9aa..0000000 --- a/icf/io.py +++ /dev/null @@ -1,666 +0,0 @@ -import struct -import binascii -from .utils import get_si_prefix -from datetime import datetime -import os -from typing import Union -import numpy as np -import bz2 - - -class IndexedContainerWriter: - """ Acts as a file object for writing chunks of serialized data - to file. Prepends each chunk with: - chunk length in bytes (4 bytes) - and a crc32 hash (4 bytes) - """ - - _protocols = {} - - def __init__(self, filename: str, protocol=1, compressor="bz2", **kwargs): - self._writer = IndexedContainerWriter._protocols[protocol]( - filename, compressor=compressor, **kwargs - ) - self.protocol = protocol - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._writer.close() - - def write(self, data: bytes): - """ Writes a stream of bytes to file - - args: - data (bytes): bytes to be writen to file - """ - self._writer.write(data) - - def close(self): - self._writer.close() - - @classmethod - def _register(cls, scls): - cls._protocols[scls._protocol_v] = scls - return scls - - @property - def data_counter(self): - return self._writer.data_counter - - -@IndexedContainerWriter._register -class IndexedContainerWriterV0: - """Acts as a file object for writing chunks of serialized data - to file. Prepends each chunk with: - chunk length in bytes (4 bytes) - and a crc32 hash (4 bytes) - - The file header is 24 bytes long and has the following layout - - bytes: field: - 0-7 Custom field for file format specifications (set by the header parameter) - 8-11 Protocol version - 12-15 Not used - 16-19 Not used - 20-23 Not used - - General file structure: - +-------------+ - | File Header | - +-------------+ - | Chunk Header| - +-------------+ - | Data | - +-------------+ - | Chunk Header| - +-------------+ - | Data | - +-------------+ - ... - ... - - Attributes: - data_counter (int): Description - file (object): Description - filename (str): Description - version (int): Description - - """ - - _protocol_v = 0 - _chunk_header = struct.Struct("<2I") - _file_header = struct.Struct(" 0: - self._write(header_ext) - self._buffer = [] - self._cbunchindex = [] - self._cbunchoffset = 0 - self._last_bunch_fp = 0 - self._bunch_number = 0 - - def _write(self, data: bytes): - self._file.write(data) - self._fp += len(data) - - def write(self, data: bytes): - """ Writes a stream of bytes to file - - args: - data (bytes): bytes to be writen to file - """ - - self._buffer.append(data) - self._cbunchindex.append((binascii.crc32(data), len(data))) - self._cbunchoffset += len(data) - self.data_counter += 1 - if self._cbunchoffset > self.bunchsize: - self.flush() - - def flush(self): - """Flushes any data in buffer to file. - - """ - if len(self._buffer) < 1: - return - bunch_start_fp = self._fp - # writing the data bunch - byte_buff = bytearray() - for data in self._buffer: - byte_buff.extend(data) - bunch_crc = binascii.crc32(data) - if self.compress: - self._write(self.compressor.compress(byte_buff)) - else: - self._write(byte_buff) - - # constructing the index and writing it in the bunch trailer - index = list(zip(*self._cbunchindex)) - n = len(self._buffer) - bunch_index = struct.pack("{}I{}I".format(n, n), *index[0], *index[1]) - self._write(bunch_index) - - bunch_index_trailer = self._bunch_trailer_header.pack( - self._fp - self._last_bunch_fp, - self._fp - bunch_start_fp, - self._fp, - bunch_crc, - len(self._buffer), - self._bunch_number, - ) - - # before writing the bunch trailer we update - # the file pointer for the last bunch - self._last_bunch_fp = self._fp - - self._write(bunch_index_trailer) - self._file.flush() - # reseting/updating the last bunch descriptors - self._cbunchindex.clear() - self._buffer.clear() - self._cbunchoffset = 0 - self._bunch_number += 1 - - def close(self): - self.flush() - self._file.close() - - -class IndexedContainerReader: - - """Summary - - Attributes: - file (TYPE): file handle - filename (TYPE): the full filename - metadata (dict): Dictonary containing file meta data - """ - - _protocols = {} - - def __init__(self, filename: str): - """ Reads Streamed Object Files - - Args: - filename (str): filename and path to the file - - Raises: - TypeError: Raised if the file is not recognized as SOF - """ - - self.filename = filename - self.file = open(self.filename, "rb") - self._fhead = self.file.read(12) # _file_header.size) - self.file.seek(0) - self.metadata = {} - - self.fhead, self.version = struct.unpack("QI", self._fhead) - - if self.version == 0 and self.fhead >= 0: - readerclass = IndexedContainerReader._protocols[self.version] - self._reader = readerclass(self.file) - elif self.version > 0: - readerclass = IndexedContainerReader._protocols[self.version] - self._fhead = self.file.read(readerclass._file_header.size) - self._reader = readerclass(self.file) - else: - raise TypeError("This file appears not to be a stream object file (SOF)") - - @classmethod - def _register(cls, scls): - cls._protocols[scls._protocol_v] = scls - return scls - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.file.close() - - def reload(self): - """ Reload the index table. Useful if the file - is being written too when read - """ - self._reader.reload() - - def resetfp(self): - """ Resets file pointer to the first object in file - """ - self._reader.resetfp() - - def __getitem__(self, ind: Union[int, slice, list]) -> bytes: - """Indexing interface to the streamed file. - Objects are read by their index, slice or list of indices. - - Args: - ind (Union[int, slice, list]): an integer index, slice or list off indices to be read - - Returns: - bytes: Bytes that represent the read object - """ - if isinstance(ind, slice): - data = [self.read_at(ii) for ii in range(*ind.indices(self.n_entries))] - return data - elif isinstance(ind, list): - data = [self.read_at(ii) for ii in ind] - return data - elif isinstance(ind, int): - return self.read_at(ind) - - def read_at(self, ind: int) -> bytes: - """Reads one object at the index indicated by `ind` - - Args: - ind (int): the index of the object to be read - - Returns: - bytes: that represent the object - - Raises: - IndexError: if index out of range - """ - return self._reader.read_at(ind) - - def read(self) -> bytes: - """ Reads one object at the position of the file pointer. - - Returns: - bytes: Bytes that represent the object - """ - return self._reader.read() - - def close(self): - """Summary - """ - self.file.close() - - @property - def n_entries(self): - """ Number of entries in file - Returns: - int: Number of entries (objects) in file - """ - return self._reader.n_entries - - @property - def filesize(self): - """Summary - - Returns: - int: Description - """ - return self._reader.filesize - - @property - def timestamp(self): - """A timestamp from the creation of the file - - Returns: - int: unix timestamp - """ - return datetime.fromtimestamp(self._reader._timestamp) - - def __str__(self): - - s = "{}:\n".format(self.__class__.__name__) - s += "filename: {}\n".format(self.filename) - s += "timestamp: {}\n".format(self.timestamp) - s += "n_entries: {}\n".format(self._reader.n_entries) - s += "file size: {} {}B\n".format(*get_si_prefix(self._reader.filesize)) - for k, v in self.metadata.items(): - s += "{}: {}\n".format(k, v) - s += "file format version: {}".format(self.version) - return s - - -@IndexedContainerReader._register -class IndexedContainerReaderV1: - - """Summary - - Attributes: - file (TYPE): Description - file_index (list): Description - filesize (TYPE): Description - n_bunches (TYPE): Description - n_entries (TYPE): Description - """ - - _protocol_v = 1 - _file_header = IndexedContainerWriterV1._file_header - _compressors = IndexedContainerWriterV1._compressors - _bunch_trailer_header = IndexedContainerWriterV1._bunch_trailer_header - - def __init__(self, file): - """Summary - - Args: - file (TYPE): Description - - Raises: - TypeError: Description - """ - fileheader_def = self._file_header - self.file = file - self.file.seek(0) - fileheader = self.file.read(fileheader_def.size) - ( - marker, - extmarker, - self._version, - self._timestamp, - self._compressed, - self._lenheadext, - ) = fileheader_def.unpack(fileheader) - if marker[:3].decode() != "SOF": - raise TypeError("This file appears not to be a stream object file (SOF)") - if self._version != self._protocol_v: - raise TypeError( - "This file is written with protocol V{}" - " while this class reads protocol V{}".format( - self._version, self._protocol_v - ) - ) - self._compressor = None - if self._compressed > 0: - compressorsr = {} - for k, v in self._compressors.items(): - compressorsr[v[1]] = (v[0], k) - self._compressor = compressorsr[self._compressed][0] - self._compressor_name = compressorsr[self._compressed][1] - self._headext = None - if self._lenheadext > 0: - self._headext = self.file.read(self._lenheadext) - self._fp_start = self.file.tell() - self.n_bunches = None - self.n_entries = None - self.filesize = None - self._rawindex = {} - self._bunch_buffer = {} - self._current_index = 0 - self._scan_file() - - def _scan_file(self): - from collections import namedtuple - - BunchTrailer = namedtuple( - "BunchTrailer", "bunchoff dataoff fileoff crc bunchsize ndata index objsize" - ) - self.file.seek(-self._bunch_trailer_header.size, os.SEEK_END) - - self.filesize = self.file.tell() - self.file_index = [0] - while self.file.tell() > self._fp_start: - # read bunch trailer - last_bunch_trailer = self.file.read(self._bunch_trailer_header.size) - ( - bunchoff, - dataoff, - fileoff, - crc, - ndata, - bunch_n, - ) = self._bunch_trailer_header.unpack(last_bunch_trailer) - - # read bunch index - self.file.seek( - self.file.tell() - self._bunch_trailer_header.size - ndata * 2 * 4 - ) - index = struct.unpack( - "{}I{}I".format(ndata, ndata), self.file.read(ndata * 2 * 4) - ) - objsize = np.array(index[ndata:], dtype=np.uint32) - self._rawindex[(0, bunch_n)] = BunchTrailer( - bunchoff, # Offset to earlier bunch or file header if first bunch - dataoff, # Offset to beginning of data in bunch - fileoff, # Offset to beginning of file - crc, # bunch crc - dataoff - ndata * 2 * 4, # Size of data bunch - ndata, # number of objects in bunch - [0] + list(np.cumsum(objsize[:-1])), # Object offsets in bunch - objsize, # object sizes - ) - self.file.seek(self.file.tell() - bunchoff) - - self._index = [] - self._bunch_index = {} - for k, bunch in sorted(self._rawindex.items()): - self._bunch_index[k] = (bunch.fileoff - bunch.dataoff, bunch.bunchsize) - for i, obj in enumerate(bunch.index): - self._index.append((k, int(obj), int(bunch.objsize[i]))) - self.n_entries = len(self._index) - - def _get_bunch(self, bunch_id): - - if bunch_id in self._bunch_buffer: - return self._bunch_buffer[bunch_id] - else: - self.file.seek(self._bunch_index[bunch_id][0]) - bunch = self._compressor.decompress( - self.file.read( - self.file_index[bunch_id[0]] + self._bunch_index[bunch_id][1] - ) - ) - self._bunch_buffer[bunch_id] = bunch - return bunch - - def read_at(self, ind: int) -> bytes: - """Reads one object at the index indicated by `ind` - - Args: - ind (int): the index of the object to be read - - Returns: - bytes: that represent the object - - Raises: - IndexError: if index out of range - """ - - if ind > self.n_entries - 1: - raise IndexError( - "The requested file object ({}) is out of range".format(ind) - ) - obji = self._index[ind] - if self._compressed: - bunch = self._get_bunch(obji[0]) - return bunch[obji[1] : obji[1] + obji[2]] - else: - fpos = self.file_index[obji[0][0]] + self._bunch_index[obji[0]][0] + obji[1] - self.file.seek(fpos) - - return self.file.read(obji[2]) - - def resetfp(self): - """ Resets file pointer to the first object in file - """ - self._current_index = 0 - - def read(self) -> bytes: - """ Reads one object at the position of the file pointer. - - Returns: - bytes: Bytes that represent the object - """ - self._current_index += 1 - return self.read_at(self._current_index - 1) - - -@IndexedContainerReader._register -class IndexedContainerReaderV0: - - _protocol_v = 0 - _chunk_header = IndexedContainerWriterV0._chunk_header - _file_header = IndexedContainerWriterV0._file_header - - def __init__(self, file): - self.file = file - self._scan_file() - self._timestamp = 0 - - def reload(self): - """ Reload the index table. Useful if the file - is being written too when read - """ - self._scan_file(self.filesize, self.n_entries, self.fpos) - - def resetfp(self): - """ Resets file pointer to the first object in file - """ - self.file.seek(IndexedContainerReaderV0._file_header.size) - - def _scan_file(self, offset=0, n_entries=0, fpos=[]): - """Summary - - Args: - offset (int, optional): Description - n_entries (int, optional): Description - fpos (list, optional): Description - """ - self.file.seek(offset) - fh = self.file - self.n_entries = n_entries - self.fpos = fpos - # Skipping file header - fp = IndexedContainerReaderV0._file_header.size - while True: - fh.seek(fp) - rd = fh.read(IndexedContainerReaderV0._chunk_header.size) - if rd == b"": - break - self.fpos.append(fp) - offset, crc = IndexedContainerReaderV0._chunk_header.unpack(rd) - self.n_entries += 1 - fp = fh.tell() + offset - self.file.seek(IndexedContainerReaderV0._file_header.size) - self.filesize = self.fpos[-1] + offset - - def read_at(self, ind: int) -> bytes: - """Reads one object at the index indicated by `ind` - - Args: - ind (int): the index of the object to be read - - Returns: - bytes: that represent the object - - Raises: - IndexError: if index out of range - """ - if ind > len(self.fpos) - 1: - raise IndexError( - "The requested file object ({}) is out of range".format(ind) - ) - self.file.seek(self.fpos[ind]) - return self.read() - - def read(self) -> bytes: - """ Reads one object at the position of the file pointer. - - Returns: - bytes: Bytes that represent the object - """ - sized = self.file.read(IndexedContainerReaderV0._chunk_header.size) - if sized == b"": - return None - size, crc = IndexedContainerReaderV0._chunk_header.unpack(sized) - return self.file.read(size) diff --git a/icf/pyicf/__init__.py b/icf/pyicf/__init__.py new file mode 100644 index 0000000..8da3b16 --- /dev/null +++ b/icf/pyicf/__init__.py @@ -0,0 +1,2 @@ +from .icffile import ICFFile +# from . import \ No newline at end of file diff --git a/icf/icffile.py b/icf/pyicf/icffile.py similarity index 85% rename from icf/icffile.py rename to icf/pyicf/icffile.py index 8dd9737..12ce30d 100644 --- a/icf/icffile.py +++ b/icf/pyicf/icffile.py @@ -6,7 +6,7 @@ import os from collections import namedtuple import numpy as np - +from icf._icf.utils import get_si_prefix class ICFFile: _file_header = struct.Struct("<4s4s2HQ2H") _bunch_trailer_header = struct.Struct("<2H4Q3I") @@ -82,6 +82,25 @@ def __init__( ) ) + def __getitem__(self, ind) -> bytes: + """Indexing interface to the streamed file. + Objects are read by their index, slice or list of indices. + + Args: + ind (Union[int, slice, list]): an integer index, slice or list off indices to be read + + Returns: + bytes: Bytes that represent the read object + """ + if isinstance(ind, slice): + data = [self.read_at(ii) for ii in range(*ind.indices(self.n_entries))] + return data + elif isinstance(ind, list): + data = [self.read_at(ii) for ii in ind] + return data + elif isinstance(ind, int): + return self.read_at(ind) + def __enter__(self): return self @@ -250,6 +269,26 @@ def read_at(self, ind: int) -> bytes: # return self.file.read(obji[2]) + def read(self) -> bytes: + """ Reads one object at the position of the file pointer. + + Returns: + bytes: Bytes that represent the object + """ + self._current_index += 1 + return self.read_at(self._current_index - 1) + + def __str__(self): + + s = "{}:\n".format(self.__class__.__name__) + s += "filename: {}\n".format(self.filename) + s += "timestamp: {}\n".format(self.timestamp) + s += "n_entries: {}\n".format(self._reader.n_entries) + s += "file size: {} {}B\n".format(*get_si_prefix(self._reader.filesize)) + for k, v in self.metadata.items(): + s += "{}: {}\n".format(k, v) + s += "file format version: {}".format(self.version) + return s from collections import deque diff --git a/pytest/test_icffile.py b/pytest/test_icffile.py index e6e3c92..0503e73 100644 --- a/pytest/test_icffile.py +++ b/pytest/test_icffile.py @@ -1,8 +1,8 @@ -from icf._icf import icffile +from icf import pyicf -def test_write_and_readback_icffile(): - f = icffile.ICFFile("/tmp/test.icf",'trunc') +def test_write_and_readback_from_icffile(): + f = pyicf.ICFFile("/tmp/test.icf",'trunc') testdata1 = b'blablakaskdlaskd' testdata2 = b'blabasdasdlakaskdlaskd' f.write(testdata1) @@ -11,8 +11,22 @@ def test_write_and_readback_icffile(): assert f.n_entries == 2, "Correct number of entries" f.close() # open file again - f = icffile.ICFFile("/tmp/test.icf") + f = pyicf.ICFFile("/tmp/test.icf") assert f.get_timestamp() == original_timestamp, "Read back correct timestamp" assert f.n_entries == 2, "Read back correct number of entries" assert f.read_at(0) == testdata1, "Read back correct data" assert f.read_at(1) == testdata2, "Read back correct data" + + +def test_bunch_buffer(): + n = 10 + bf = pyicf.icffile.BunchBuffer(n) + + for i in range(n): + bf[i] = [i] + for i in range(n): + assert i in bf, "Element still in Bunch Buffer" + + bf[n] = [n] + assert n in bf, "New element in Bunch Buffer" + assert 0 not in bf, "Oldest element removed from Bunch Buffer" From 5a769317ec56936dff5275715d547e4905e40476 Mon Sep 17 00:00:00 2001 From: Samuel Date: Sun, 16 Feb 2020 23:36:57 +0100 Subject: [PATCH 11/22] Fixed wrong seek in file scan --- icf/__init__.py | 4 ++-- icf/pyicf/icffile.py | 30 ++++++++++++++++------------ pytest/test_icffile.py | 45 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/icf/__init__.py b/icf/__init__.py index 6b0f7ec..9da05b4 100644 --- a/icf/__init__.py +++ b/icf/__init__.py @@ -1,4 +1,4 @@ -from icf._icf import frame, pyicf - +# from icf._icf import frame, pyicf +from . import frame, pyicf from .frame import Frame diff --git a/icf/pyicf/icffile.py b/icf/pyicf/icffile.py index 12ce30d..2701b2b 100644 --- a/icf/pyicf/icffile.py +++ b/icf/pyicf/icffile.py @@ -1,12 +1,11 @@ import struct - -# import binascii -# from .utils import get_si_prefix +from collections import deque, namedtuple from datetime import datetime import os -from collections import namedtuple import numpy as np from icf._icf.utils import get_si_prefix + + class ICFFile: _file_header = struct.Struct("<4s4s2HQ2H") _bunch_trailer_header = struct.Struct("<2H4Q3I") @@ -37,6 +36,7 @@ def __init__( self._bunch_number = 0 self._bunch_buffer = BunchBuffer(10) self._file_index = [0] + omode = "b" if mode == "append": omode += "a+" @@ -164,7 +164,7 @@ def flush(self): self._write(bunch_index) # Write offset to begining of bunch trailer - self._write(struct.pack("I", self._file.tell() - curr_bt_fp)) + self._write(struct.pack("= pos_start and curr_bunch > 0: @@ -202,8 +203,9 @@ def _scan_file(self, pos_start, pos_end): bunch_n, flags ) = self._bunch_trailer_header.unpack(last_bunch_trailer) + curr_bunch = bunch_n - index = struct.unpack("{}I".format(ndata), self._file.read(ndata * 4)) + index = struct.unpack("<{}I".format(ndata), self._file.read(ndata * 4)) objsizes = np.array(index, dtype=np.uint32) rawindex[(0, bunch_n)] = self._BunchTrailer( bunchoff, # Offset to earlier bunch or file header if first bunch @@ -214,8 +216,10 @@ def _scan_file(self, pos_start, pos_end): [0] + list(np.cumsum(objsizes[:-1])), # Object offsets in bunch objsizes, # object sizes ) - self._file.seek(self._file.tell() - bunchoff) - print('Ndata',ndata,self._file.tell(),pos_start,curr_bunch) + current_bt_fp -= bunchoff + self._file.seek(current_bt_fp) + + print('Ndata',ndata,self._file.tell(),pos_start,curr_bunch,bunchoff) return rawindex def _construct_file_index(self, rawindex): @@ -227,7 +231,6 @@ def _construct_file_index(self, rawindex): self._index.append((k, int(obj), int(bunch.objsize[i]))) self.n_entries = len(self._index) - def _get_bunch(self, bunch_id): if bunch_id in self._bunch_buffer: @@ -257,7 +260,7 @@ def read_at(self, ind: int) -> bytes: if ind > self.n_entries - 1: raise IndexError( - "The requested file object ({}) is out of range".format(ind) + "The requested file object at index ({}) is out of range".format(ind) ) obji = self._index[ind] if True: # self._compressed: @@ -290,7 +293,8 @@ def __str__(self): s += "file format version: {}".format(self.version) return s -from collections import deque + def size(self): + return self.n_entries class BunchBuffer(dict): @@ -303,4 +307,4 @@ def __setitem__(self, key, obj): super().__setitem__(key, obj) if len(self.queue) > self.size: bunch = self.queue.popleft() - super().pop(bunch[0]) \ No newline at end of file + super().pop(bunch[0]) diff --git a/pytest/test_icffile.py b/pytest/test_icffile.py index 0503e73..0b344b2 100644 --- a/pytest/test_icffile.py +++ b/pytest/test_icffile.py @@ -1,23 +1,58 @@ +import pytest from icf import pyicf +from icf import ICFFile +import os +@pytest.fixture(params=[ICFFile, pyicf.ICFFile]) +def icf_impl(request): + return request.param -def test_write_and_readback_from_icffile(): - f = pyicf.ICFFile("/tmp/test.icf",'trunc') + +def test_write_and_readback_from_icffile(icf_impl): + os.remove("/tmp/test.icf") + f = icf_impl("/tmp/test.icf") testdata1 = b'blablakaskdlaskd' testdata2 = b'blabasdasdlakaskdlaskd' f.write(testdata1) f.write(testdata2) original_timestamp = f.get_timestamp() - assert f.n_entries == 2, "Correct number of entries" + assert f.size() == 2, "Correct number of entries" f.close() # open file again - f = pyicf.ICFFile("/tmp/test.icf") + f = icf_impl("/tmp/test.icf") assert f.get_timestamp() == original_timestamp, "Read back correct timestamp" - assert f.n_entries == 2, "Read back correct number of entries" + assert f.size() == 2, "Read back correct number of entries" + assert f.read_at(0) == testdata1, "Read back correct data" + assert f.read_at(1) == testdata2, "Read back correct data" + +def test_read_while_writing(icf_impl): + os.remove("/tmp/test.icf") + f = icf_impl("/tmp/test.icf") + testdata1 = b'blablakaskdlaskd' + testdata2 = b'blabasdasdlakaskdlaskd' + f.write(testdata1) + f.write(testdata2) assert f.read_at(0) == testdata1, "Read back correct data" assert f.read_at(1) == testdata2, "Read back correct data" +def test_write_read_multiple_bunches(icf_impl): + os.remove("/tmp/test.icf") + f = pyicf.ICFFile("/tmp/test.icf",bunchsize=50) + data = b"0"*26 + + for i in range(6): + f.write(data) + assert f._bunch_number == 3, "Correct number of bunches" + f.close() + + f = pyicf.ICFFile("/tmp/test.icf") + + for i in range(6): + assert f.read_at(i) == data + + + def test_bunch_buffer(): n = 10 bf = pyicf.icffile.BunchBuffer(n) From be4f921b4c9524f8693c4d1bca4f45de35e94732 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 18 Feb 2020 15:35:11 +0100 Subject: [PATCH 12/22] Add RPath fix for custom install locations and add more tests --- CMakeLists.txt | 20 ++++++++++++++++++++ pytest/test_icffile.py | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb9fd74..d79f441 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,26 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) + # The following settings were copied from + # https://cmake.org/Wiki/CMake_RPATH_handling + # to avoid the rpath issue (issue #11998) that appears on OS X El Capitan + # https://forge.in2p3.fr/issues/11998 + + # use, i.e. don't skip the full RPATH for the build tree + set(CMAKE_SKIP_BUILD_RPATH FALSE) + + # when building, don't use the install RPATH already + # (but later on when installing) + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) # Changed to TRUE by A.O. + + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + + # add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +endif() + find_package(Git QUIET) find_package(Python3 COMPONENTS Interpreter) diff --git a/pytest/test_icffile.py b/pytest/test_icffile.py index 0b344b2..dcf8a2c 100644 --- a/pytest/test_icffile.py +++ b/pytest/test_icffile.py @@ -9,7 +9,10 @@ def icf_impl(request): def test_write_and_readback_from_icffile(icf_impl): - os.remove("/tmp/test.icf") + try: + os.remove("/tmp/test.icf") + except: + pass f = icf_impl("/tmp/test.icf") testdata1 = b'blablakaskdlaskd' testdata2 = b'blabasdasdlakaskdlaskd' @@ -26,7 +29,10 @@ def test_write_and_readback_from_icffile(icf_impl): assert f.read_at(1) == testdata2, "Read back correct data" def test_read_while_writing(icf_impl): - os.remove("/tmp/test.icf") + try: + os.remove("/tmp/test.icf") + except: + pass f = icf_impl("/tmp/test.icf") testdata1 = b'blablakaskdlaskd' testdata2 = b'blabasdasdlakaskdlaskd' @@ -37,7 +43,10 @@ def test_read_while_writing(icf_impl): def test_write_read_multiple_bunches(icf_impl): - os.remove("/tmp/test.icf") + try: + os.remove("/tmp/test.icf") + except: + pass f = pyicf.ICFFile("/tmp/test.icf",bunchsize=50) data = b"0"*26 @@ -51,6 +60,28 @@ def test_write_read_multiple_bunches(icf_impl): for i in range(6): assert f.read_at(i) == data +def test_read_merged_files(icf_impl): + try: + os.remove("/tmp/test1.icf") + os.remove("/tmp/test2.icf") + os.remove("/tmp/cat.icf") + except: + pass + f1 = icf_impl("/tmp/test1.icf") + f2 = icf_impl("/tmp/test2.icf") + testdata1 = b'blablakaskdlaskd' + testdata2 = b'blabasdasdlakaskdlaskd' + f1.write(testdata1) + f2.write(testdata2) + f1.close() + f2.close() + os.system("cat /tmp/test1.icf /tmp/test2.icf >> /tmp/cat.icf") + fcat = icf_impl("/tmp/cat.icf") + + assert fcat.size() == 2, "Correct number of elements" + assert fcat.read_at(0) == testdata1, "Correct data from file 1" + assert fcat.read_at(1) == testdata2, "Correct data from file 2" + def test_bunch_buffer(): @@ -65,3 +96,6 @@ def test_bunch_buffer(): bf[n] = [n] assert n in bf, "New element in Bunch Buffer" assert 0 not in bf, "Oldest element removed from Bunch Buffer" + + + From eaaff188e104ea434e30645b71c1c842e4d23d2d Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 20 Feb 2020 23:26:37 +0100 Subject: [PATCH 13/22] CPP version of ICF now basically writes according to the specification implemented by the python version --- CMakeLists.txt | 2 +- include/icf/icfFile.h | 72 ++++++++------------- src/icfFile.cc | 142 +++++++++++++++++++----------------------- 3 files changed, 91 insertions(+), 125 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 882ff88..c0bd375 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) # when building, don't use the install RPATH already # (but later on when installing) - set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) # Changed to TRUE by A.O. + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # Changed to TRUE by A.O. set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") diff --git a/include/icf/icfFile.h b/include/icf/icfFile.h index 95b8caf..e552ee5 100644 --- a/include/icf/icfFile.h +++ b/include/icf/icfFile.h @@ -27,8 +27,7 @@ namespace icf{ class ICFFile{ -public: - struct ICFFileHeader{ + struct ICFFileHeader{ ICFFileHeader():version_(0),compression_(0),unused_(0),ext_head_len_(0){ strncpy(file_identifier_,"ICF",4); @@ -62,54 +61,30 @@ class ICFFile{ uint16_t ext_head_len_; }; - - enum access_mode{read, trunc, append}; - - ICFFile(std::string path, ICFFile::access_mode mode=ICFFile::append); - - ~ICFFile(); - - void write(const void* data, std::size_t size); - - std::shared_ptr > read_at(uint64_t index); - - void flush(); - - void close(){file_handle_.close(); } - - uint64_t size(){return object_index_.size();} - - std::chrono::time_point get_timestamp(){return file_header_.time_stamp2_;} -private: - - void scan_file(uint64_t pos); - - ICFFileHeader file_header_; - std::fstream file_handle_; - uint64_t current_read_pointer_; - uint64_t current_write_pointer_; - std::vector object_index_; - Archive serializer_stream_; -}; - - - - -class ICFFileV2{ struct ICFBunchTrailer{ - ICFBunchTrailer(std::weak_ptr >):version_(0){ + ICFBunchTrailer(uint64_t file_offset, + uint64_t prev_bunch_offset, + uint64_t bunch_data_offset, + uint32_t n_chunks_in_bunch, + uint32_t bunch_number):file_offset_(file_offset), + prev_bunch_offset_(prev_bunch_offset), + bunch_data_offset_(bunch_data_offset), + n_chunks_in_bunch_(n_chunks_in_bunch), + bunch_number_(bunch_number), + version_(0){ time_stamp2_ = std::chrono::system_clock::now(); time_stamp_ = std::chrono::system_clock::to_time_t(time_stamp2_); } - void Serialize(Archive& archive) + template + void Serialize(T& archive) { archive & version_; archive & unused_ & - time_stamp_ & + time_stamp_ & file_offset_ & prev_bunch_offset_ & bunch_data_offset_ & @@ -118,7 +93,7 @@ class ICFFileV2{ flags_; } - + uint16_t version_; uint16_t unused_; std::time_t time_stamp_; @@ -130,16 +105,16 @@ class ICFFileV2{ uint32_t bunch_number_; uint32_t flags_; - uint32_t trailer_start_offset_; + // uint32_t trailer_start_offset_; }; public: enum access_mode{read, trunc, append}; - ICFFileV2(std::string path, ICFFileV2::access_mode mode=ICFFileV2::append); + ICFFile(std::string path, ICFFile::access_mode mode=ICFFile::append, uint32_t bunchsize = 1000000); - ~ICFFileV2(); + ~ICFFile(); void write(const void* data, std::size_t size); @@ -147,11 +122,13 @@ class ICFFileV2{ void flush(); - void close(){file_handle_.close(); } + void close(){flush();file_handle_.close(); } uint64_t size(){return object_index_.size();} std::chrono::time_point get_timestamp(){return file_header_.time_stamp2_;} + + private: void scan_file(uint64_t pos); @@ -162,7 +139,12 @@ class ICFFileV2{ uint64_t current_write_pointer_; std::vector object_index_; Archive serializer_stream_; - std::vector > write_buffer_; + std::vector > > write_buffer_; + uint32_t n_entries_; + uint32_t cbunchoffset_; + uint32_t bunchsize_; + uint64_t last_bunch_fp_; + uint32_t bunch_number_; // std::vector > read_buffer_; }; diff --git a/src/icfFile.cc b/src/icfFile.cc index 8909f33..3270a55 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -1,11 +1,21 @@ #include "icf/icfFile.h" #include #include - +#include using namespace icf; -ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_(file_handle_){ + + + + +ICFFile::ICFFile(std::string path, ICFFile::access_mode mode, uint32_t bunchsize): + serializer_stream_(file_handle_), + n_entries_(0), + cbunchoffset_(0), + bunchsize_(bunchsize), + last_bunch_fp_(0), + bunch_number_(0){ auto omode = std::ios_base::in | std::ios_base::binary; switch(mode){ case append: @@ -27,13 +37,14 @@ ICFFile::ICFFile(std::string path, ICFFile::access_mode mode):serializer_stream_ if(current_write_pointer_>current_read_pointer_ && mode !=trunc){ serializer_stream_>>file_header_; - scan_file(file_handle_.tellp()); + scan_file(file_handle_.tellp()); } else{ serializer_stream_<>bt; } ICFFile::~ICFFile(){ @@ -41,92 +52,65 @@ ICFFile::~ICFFile(){ } - -void ICFFile::write(const void* data, std::size_t size){ - file_handle_.seekp(current_write_pointer_); - object_index_.push_back(file_handle_.tellp()); - serializer_stream_< > ICFFile::read_at(uint64_t index){ - - auto fp = object_index_.at(index); - size_t obj_size; - file_handle_.seekp(fp); - serializer_stream_>>obj_size; - auto data = std::make_shared >(obj_size); - - file_handle_.read((char*) data->data(), obj_size); - return data; -} - - void ICFFile::scan_file(uint64_t pos){ - size_t obj_size; - auto bheader_size = sizeof(obj_size); - auto length = current_write_pointer_; - file_handle_.seekp(pos); - while(file_handle_.tellp()>obj_size; - pos += obj_size + bheader_size; - file_handle_.seekp(pos); - } + // size_t obj_size; + // auto bheader_size = sizeof(obj_size); + // auto length = current_write_pointer_; + // file_handle_.seekp(pos); + // while(file_handle_.tellp()>obj_size; + // pos += obj_size + bheader_size; + // file_handle_.seekp(pos); + // } } - - - -ICFFileV2::ICFFileV2(std::string path, ICFFileV2::access_mode mode):serializer_stream_(file_handle_){ - auto omode = std::ios_base::in | std::ios_base::binary; - switch(mode){ - case append: - omode |= std::ios_base::app | std::ios_base::out; - break; - case trunc: - omode |= std::ios_base::trunc | std::ios_base::out; - default: - break; +void ICFFile::write(const void* data, std::size_t size){ + auto datacopy = std::make_shared >(std::vector(size)); + memcpy(datacopy->data(),data, size); + write_buffer_.push_back(datacopy); + n_entries_ +=1; + cbunchoffset_ += size; + if(cbunchoffset_> bunchsize_){ + flush(); } - file_handle_.open(path, omode); +} - current_read_pointer_ = file_handle_.tellg(); +void ICFFile::flush(){ + cbunchoffset_ = 0; + if(write_buffer_.size()<1){ + return; + } file_handle_.seekp(0, std::ios_base::end); - current_write_pointer_ = file_handle_.tellp(); - auto length = current_write_pointer_; - file_handle_.seekp(0); - - if(current_write_pointer_>current_read_pointer_ && mode !=trunc){ - serializer_stream_>>file_header_; - scan_file(file_handle_.tellp()); + auto bunch_start_fp = file_handle_.tellp(); + std::vector cbunchindex; + // cbunchindex_ + for(auto &data: write_buffer_){ + cbunchindex.push_back(data->size()); + file_handle_.write(data->data(),data->size()); } - else{ - serializer_stream_<>bt; -} + uint32_t offset_to_bt_start = file_handle_.tellp() - curr_bt_fp; + serializer_stream_<>obj_size; - pos += obj_size + bheader_size; - file_handle_.seekp(pos); - } +std::shared_ptr > ICFFile::read_at(uint64_t index){ + + } \ No newline at end of file From 70b2db958ec596809b3598376bfe93afb386848f Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 20 Feb 2020 23:39:26 +0100 Subject: [PATCH 14/22] Minor fix to remove compiler warning --- src/icfFile.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/icfFile.cc b/src/icfFile.cc index 3270a55..068fd88 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -82,18 +82,19 @@ void ICFFile::flush(){ if(write_buffer_.size()<1){ return; } + file_handle_.seekp(0, std::ios_base::end); auto bunch_start_fp = file_handle_.tellp(); std::vector cbunchindex; - // cbunchindex_ + for(auto &data: write_buffer_){ cbunchindex.push_back(data->size()); file_handle_.write(data->data(),data->size()); } auto curr_bt_fp = file_handle_.tellp(); ICFFile::ICFBunchTrailer bunch_trailer(curr_bt_fp, - curr_bt_fp-last_bunch_fp_, - curr_bt_fp-bunch_start_fp, + static_cast(curr_bt_fp) - last_bunch_fp_, + curr_bt_fp - bunch_start_fp, write_buffer_.size(), bunch_number_); serializer_stream_< Date: Fri, 21 Feb 2020 18:54:30 +0100 Subject: [PATCH 15/22] Added test that tests the consistency between the c++ and python implementation --- pytest/test_icffile.py | 21 +++++++++++++++++++++ src/icfFile.cc | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pytest/test_icffile.py b/pytest/test_icffile.py index 3e04470..5b08fd4 100644 --- a/pytest/test_icffile.py +++ b/pytest/test_icffile.py @@ -116,6 +116,27 @@ def test_read_merged_files_multiple_bunches(icf_impl): assert fcat.read_at(3) == testdata2, "Correct data from file 2" +def test_extwrie_pyread(): + try: + os.remove("/tmp/ext.icf") + except: + pass + f = ICFFile("/tmp/ext.icf") + testdata1 = b'blablakaskdlaskd' + testdata2 = b'blabasdasdlakaskdlaskd' + f.write(testdata1) + f.write(testdata2) + timestamp = f.get_timestamp() + f.close() + + f_py = pyicf.ICFFile("/tmp/ext.icf") + + assert f_py.get_timestamp() == timestamp, "Consistent time stamp" + assert f_py.size() == 2, "Consistent number of chunks" + assert f_py.read_at(0) == testdata1, "Correct data written in ext and read in py" + assert f_py.read_at(1) == testdata2, "Correct data written in ext and read in py" + + def test_bunch_buffer(): n = 10 bf = pyicf.icffile.BunchBuffer(n) diff --git a/src/icfFile.cc b/src/icfFile.cc index 068fd88..3f4017f 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -78,7 +78,7 @@ void ICFFile::write(const void* data, std::size_t size){ } void ICFFile::flush(){ - cbunchoffset_ = 0; + if(write_buffer_.size()<1){ return; } @@ -101,8 +101,11 @@ void ICFFile::flush(){ for(const auto &index: cbunchindex){ serializer_stream_< Date: Sat, 22 Feb 2020 00:58:34 +0100 Subject: [PATCH 16/22] pip install almost works, still need to figure out how to handle the icf library --- CMakeLists.txt | 12 +++- icf/__init__.py | 11 +++- setup.py | 168 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 6 deletions(-) create mode 100644 setup.py diff --git a/CMakeLists.txt b/CMakeLists.txt index c0bd375..bef79fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.11...3.16) project(icf VERSION 1.0 LANGUAGES CXX) +option(PYTHON_SETUP "If invoked by setup.py" OFF) set(LIBTARGET ${PROJECT_NAME}) set(PYTARGET ${PROJECT_NAME}_py) set(CMAKE_CXX_STANDARD 17) @@ -18,7 +19,7 @@ if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) # when building, don't use the install RPATH already # (but later on when installing) - set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # Changed to TRUE by A.O. + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) # Changed to TRUE by A.O. set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") @@ -89,6 +90,8 @@ include_directories( ${CMAKE_BINARY_DIR}/generated/) # pybind +# set(PỲTARGET_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/") +# message(${PYTARGET_OUTPUT_DIR}) pybind11_add_module(${PYTARGET} pybind/module.cc pybind/icfFile.cc) target_link_libraries(${PYTARGET} PRIVATE ${LIBTARGET}) set_target_properties(${PYTARGET} @@ -96,7 +99,7 @@ set_target_properties(${PYTARGET} PREFIX "" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/" ) - +if(NOT PYTHON_SETUP) # Python stuff # Creating a symlink to the python package add_custom_command(TARGET ${PYTARGET} POST_BUILD @@ -110,7 +113,9 @@ add_custom_command(TARGET ${PYTARGET} POST_BUILD if(Python3_Interpreter_FOUND) add_test(NAME PythonTests COMMAND Python3::Interpreter -m pytest -r a -v WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pytest") endif() -configure_file(setup.py.in "${PROJECT_BINARY_DIR}/setup.py") + + + configure_file(setup.py.in "${PROJECT_BINARY_DIR}/setup.py") # Place the initialization file in the output directory for the Python @@ -134,3 +139,4 @@ target_link_libraries(test_ICFFile ${LIBTARGET}) target_include_directories(test_ICFFile PUBLIC ctests ${DOCTEST_INCLUDE_DIR}) target_compile_features(test_ICFFile PRIVATE cxx_std_11) +endif() \ No newline at end of file diff --git a/icf/__init__.py b/icf/__init__.py index 30c77c4..0bf962d 100644 --- a/icf/__init__.py +++ b/icf/__init__.py @@ -1,3 +1,8 @@ -# from icf._icf import frame, pyicf -from . import frame, pyicf -from .frame import Frame +from ._ext.icf_py import * +from ._ext.icf_py import _get_version + +__version__ = _get_version() + +# # from icf._icf import frame, pyicf +# from . import frame, pyicf +# from .frame import Frame diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..cc08e16 --- /dev/null +++ b/setup.py @@ -0,0 +1,168 @@ +import os +import pathlib + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as build_ext_orig + + +# class CMakeExtension(Extension): + +# def __init__(self, name): +# # don't invoke the original build_ext for this special extension +# super().__init__(name, sources=[]) + + +# class build_ext(build_ext_orig): + +# def run(self): +# for ext in self.extensions: +# self.build_cmake(ext) +# super().run() + +# def build_cmake(self, ext): +# cwd = pathlib.Path().absolute() + +# # these dirs will be created in build_py, so if you don't have +# # any python sources to bundle, the dirs will be missing +# build_temp = pathlib.Path(self.build_temp) +# build_temp.mkdir(parents=True, exist_ok=True) +# extdir = pathlib.Path(self.get_ext_fullpath(ext.name)) +# extdir.mkdir(parents=True, exist_ok=True) + +# # example of cmake args +# config = 'Debug' if self.debug else 'Release' +# cmake_args = [ +# # '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(extdir.parent.absolute()), +# '-DCMAKE_BUILD_TYPE=' + config, +# '-DPYTARGET_OUTPUT_DIR='+ str(extdir.parent.absolute()), +# '-DPYTHON_SETUP=ON' +# ] + +# # example of build args +# build_args = [ +# '--config', config, +# '--', '-j4', +# ] + +# os.chdir(str(build_temp)) +# self.spawn(['cmake', str(cwd)] + cmake_args) +# if not self.dry_run: +# self.spawn(['cmake', '--build', '.'] + build_args) +# os.chdir(str(cwd)) + +import os +import re +import sys +import sysconfig +import platform +import subprocess +from pathlib import Path + +from distutils.version import LooseVersion +from setuptools import setup, Extension, find_packages +from setuptools.command.build_ext import build_ext +from setuptools.command.test import test as TestCommand + + +class CMakeExtension(Extension): + def __init__(self, name): + Extension.__init__(self, name, sources=[]) + + +class CMakeBuild(build_ext): + def run(self): + try: + out = subprocess.check_output(['cmake', '--version']) + except OSError: + raise RuntimeError( + "CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions)) + env = os.environ.copy() + build_directory = os.path.abspath(self.build_temp) + + cmake_args = [ + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + build_directory, + '-DPYTHON_EXECUTABLE=' + sys.executable, + '-DCMAKE_INSTALL_PREFIX=' + env['CONDA_PREFIX'] + ] + + cfg = 'Debug' if self.debug else 'Release' + build_args = ['--config', cfg] + + cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + + # Assuming Makefiles + build_args += ['--', '-j2'] + + self.build_args = build_args + + + # env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format( + # env.get('CXXFLAGS', ''), + # self.distribution.get_version()) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + # CMakeLists.txt is in the same directory as this setup.py file + cmake_list_dir = os.path.abspath(os.path.dirname(__file__)) + print('-'*10, 'Running CMake prepare', '-'*40) + subprocess.check_call(['cmake', cmake_list_dir] + cmake_args, + cwd=self.build_temp, env=env) + + print('-'*10, 'Building extensions', '-'*40) + cmake_cmd = ['cmake', '--build', '.'] + self.build_args + subprocess.check_call(cmake_cmd, + cwd=self.build_temp) + + print('-'*10, 'Installing cpp library', '-'*40) + cmake_cmd = ['make', 'install'] + subprocess.check_call(cmake_cmd, + cwd=self.build_temp) + # Move from build temp to final position + for ext in self.extensions: + self.move_output(ext) + + def move_output(self, ext): + build_temp = Path(self.build_temp).resolve() + dest_path = Path(self.get_ext_fullpath(ext.name)).resolve() + source_path = build_temp / self.get_ext_filename(ext.name) + dest_directory = dest_path.parents[0] + dest_directory.mkdir(parents=True, exist_ok=True) + self.copy_file(source_path, dest_path) + + + +from setuptools import setup, find_packages + +install_requires = ["numpy"] + +setup( + name="pyicf", + + version="0.1.0", + description="An indexable container file format.", + author="Samuel Flis", + author_email="samuel.d.flis@gmail.com", + url="https://github.com/sflis/pyicf", + packages=find_packages(), + provides=["icf"], + license="GNU Lesser General Public License v3 or later", + install_requires=install_requires, + ext_modules=[CMakeExtension('icf/_ext/icf_py')], + cmdclass={ + 'build_ext': CMakeBuild, + }, + classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + # entry_points={ + # "console_scripts": [ + # ] + # }, +) \ No newline at end of file From a952a6a22af381ecd2f12df7e370650b659e536f Mon Sep 17 00:00:00 2001 From: Samuel Date: Sat, 22 Feb 2020 22:36:20 +0100 Subject: [PATCH 17/22] Remove uneccesary steps from the pip install --- CMakeLists.txt | 32 ++++++++++++++++++-------------- icf/pyicf/icffile.py | 2 +- setup.py | 3 ++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bef79fc..9b77bba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,11 @@ if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) # when building, don't use the install RPATH already # (but later on when installing) - set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) # Changed to TRUE by A.O. + if(PYTHON_SETUP) + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + else() + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + endif() set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") @@ -52,20 +56,20 @@ if(NOT pybind11_POPULATED) add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) endif() - -message(STATUS "${Cyan}doctest${ColourReset}") -FetchContent_Declare( - doctest - GIT_REPOSITORY https://github.com/onqtam/doctest - GIT_TAG 2.3.6 -) -FetchContent_GetProperties(doctest) -if(NOT doctest_POPULATED) - FetchContent_Populate(doctest) - add_subdirectory(${doctest_SOURCE_DIR} ${doctest_BINARY_DIR}) +if(NOT PYTHON_SETUP) + message(STATUS "${Cyan}doctest${ColourReset}") + FetchContent_Declare( + doctest + GIT_REPOSITORY https://github.com/onqtam/doctest + GIT_TAG 2.3.6 + ) + FetchContent_GetProperties(doctest) + if(NOT doctest_POPULATED) + FetchContent_Populate(doctest) + add_subdirectory(${doctest_SOURCE_DIR} ${doctest_BINARY_DIR}) + endif() + set(DOCTEST_INCLUDE_DIR ${doctest_SOURCE_DIR}/doctest CACHE INTERNAL "Path to include folder for doctest") endif() -set(DOCTEST_INCLUDE_DIR ${doctest_SOURCE_DIR}/doctest CACHE INTERNAL "Path to include folder for doctest") - # src add_library(${LIBTARGET} SHARED src/icfFile.cc) diff --git a/icf/pyicf/icffile.py b/icf/pyicf/icffile.py index 9a690bf..1f12a07 100644 --- a/icf/pyicf/icffile.py +++ b/icf/pyicf/icffile.py @@ -3,7 +3,7 @@ from datetime import datetime import os import numpy as np -from icf._icf.utils import get_si_prefix +from icf.utils import get_si_prefix class ICFFile: diff --git a/setup.py b/setup.py index cc08e16..0539272 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,8 @@ def run(self): cmake_args = [ '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + build_directory, '-DPYTHON_EXECUTABLE=' + sys.executable, - '-DCMAKE_INSTALL_PREFIX=' + env['CONDA_PREFIX'] + '-DCMAKE_INSTALL_PREFIX=' + env['CONDA_PREFIX'], + '-DPYTHON_SETUP=ON' ] cfg = 'Debug' if self.debug else 'Release' From 3673a3ea14d9854a57e1b6c3184f43610dbcdbef Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 6 Mar 2020 16:16:03 +0100 Subject: [PATCH 18/22] No need for setting the install prefix --- CMakeLists.txt | 50 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b77bba..c3f0842 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,12 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +if(EXISTS $ENV{CONDA_PREFIX}) + set(CMAKE_INSTALL_PREFIX $ENV{CONDA_PREFIX}) + +endif() + + if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) # The following settings were copied from # https://cmake.org/Wiki/CMake_RPATH_handling @@ -33,8 +39,9 @@ if((APPLE) OR (EXISTS $ENV{CONDA_PREFIX})) endif() find_package(Git QUIET) -find_package(Python3 COMPONENTS Interpreter) -find_package(pybind11 REQUIRED) +# find_package(Python3 COMPONENTS Interpreter) +find_package(pybind11 QUIET) + set_property(GLOBAL PROPERTY USE_FOLDERS ON) include(CTest) include(ExternalProject) @@ -44,16 +51,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) # Dependencies include(FetchContent) -FetchContent_Declare( - pybind11 - GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v2.4.3 -) -FetchContent_GetProperties(pybind11) -if(NOT pybind11_POPULATED) - FetchContent_Populate(pybind11) - add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) +if(NOT pybind11_FOUND) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11 + GIT_TAG v2.4.3 + GIT_SHALLOW TRUE + ) + + FetchContent_GetProperties(pybind11) + if(NOT pybind11_POPULATED) + FetchContent_Populate(pybind11) + add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) + endif() endif() if(NOT PYTHON_SETUP) @@ -62,15 +73,19 @@ if(NOT PYTHON_SETUP) doctest GIT_REPOSITORY https://github.com/onqtam/doctest GIT_TAG 2.3.6 + GIT_SHALLOW TRUE ) FetchContent_GetProperties(doctest) if(NOT doctest_POPULATED) FetchContent_Populate(doctest) + set(DOCTEST_WITH_TESTS OFF CACHE BOOL "No tests of doctests") + set(DOCTEST_NO_INSTALL ON CACHE BOOL "No doctest install") add_subdirectory(${doctest_SOURCE_DIR} ${doctest_BINARY_DIR}) endif() set(DOCTEST_INCLUDE_DIR ${doctest_SOURCE_DIR}/doctest CACHE INTERNAL "Path to include folder for doctest") endif() + # src add_library(${LIBTARGET} SHARED src/icfFile.cc) target_include_directories(${LIBTARGET} PUBLIC @@ -78,8 +93,17 @@ target_include_directories(${LIBTARGET} PUBLIC $ ) target_compile_features(${LIBTARGET} PUBLIC cxx_std_11) -install (TARGETS ${LIBTARGET} EXPORT icf-file-targets LIBRARY DESTINATION lib) +if(PYTHON_SETUP AND NOT EXISTS $ENV{CONDA_PREFIX}) + set_target_properties(${LIBTARGET} + PROPERTIES + PREFIX "" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/icf/_ext/" + ) +else() + install (TARGETS ${LIBTARGET} EXPORT icf-file-targets LIBRARY DESTINATION lib) + install (DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include) +endif() # Makes it easier for IDEs to find all headers source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${HEADER_LIST}) @@ -115,7 +139,7 @@ add_custom_command(TARGET ${PYTARGET} POST_BUILD if(Python3_Interpreter_FOUND) - add_test(NAME PythonTests COMMAND Python3::Interpreter -m pytest -r a -v WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pytest") + add_test(NAME PythonTests COMMAND python3 -m pytest -r a -v WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pytest") endif() From 77e9baf61156fbd52ca1d42f6a4646fa4fc376ea Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 6 Mar 2020 17:12:49 +0100 Subject: [PATCH 19/22] Fix pep8 compatible version strings --- CMakeLists.txt | 10 +++++++++- include/icf/archive.h | 22 ++++++++++++++++++++++ setup.py | 2 +- src/icfFile.cc | 1 + version_config.h.in | 6 ++---- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3f0842..edde1a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,9 +110,17 @@ source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${ #Get git version number execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=7 WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE ICF_VERSION_LIB + OUTPUT_VARIABLE VERSION ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) +set(ICF_VERSION_LONG ${VERSION}) +#parse the version information into pieces. +string(REGEX REPLACE "^v([0-9]+)\\..*" "\\1" VERSION_MAJOR "${VERSION}") +string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" VERSION_MINOR "${VERSION}") +string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.([0-9]+).*" "\\1" VERSION_PATCH "${VERSION}") +string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.[0-9]+(.*)" "\\1" VERSION_SHA1 "${VERSION}") +set(ICF_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") + configure_file( version_config.h.in ${CMAKE_BINARY_DIR}/generated/version_config.h ) include_directories( ${CMAKE_BINARY_DIR}/generated/) diff --git a/include/icf/archive.h b/include/icf/archive.h index b530751..cb5def9 100644 --- a/include/icf/archive.h +++ b/include/icf/archive.h @@ -42,6 +42,8 @@ For more information, please refer to #include #include +typedef len_t uint64_t; + namespace EndianSwapper { class SwapByteBase @@ -245,6 +247,26 @@ class Archive SERIALIZER_FOR_POD(float) SERIALIZER_FOR_POD(double) + Archive& operator&(len_t & v) + { + char first_byte; + m_stream.read(&first_byte, sizeof(char)); + if(!m_stream) { throw std::runtime_error("malformed data"); } + uint8_t number_of_bytes = (first_byte%4)*2-1; + m_stream.read((char*)&v, sizeof(number_of_bytes)); + v+= first_byte/4; + v = Swap(v); + return *this; + } + + const Archive& operator&(len_t v) const + { + v = Swap(v); + m_stream.write((const char*)&v, sizeof(len_t )); + return *this; + } + + #define SERIALIZER_FOR_STL(type) \ template \ diff --git a/setup.py b/setup.py index 0539272..500d4b2 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): cmake_args = [ '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + build_directory, '-DPYTHON_EXECUTABLE=' + sys.executable, - '-DCMAKE_INSTALL_PREFIX=' + env['CONDA_PREFIX'], + # '-DCMAKE_INSTALL_PREFIX=' + env['CONDA_PREFIX'], '-DPYTHON_SETUP=ON' ] diff --git a/src/icfFile.cc b/src/icfFile.cc index 3f4017f..2bfca52 100644 --- a/src/icfFile.cc +++ b/src/icfFile.cc @@ -71,6 +71,7 @@ void ICFFile::write(const void* data, std::size_t size){ write_buffer_.push_back(datacopy); n_entries_ +=1; cbunchoffset_ += size; + if(cbunchoffset_> bunchsize_){ flush(); } diff --git a/version_config.h.in b/version_config.h.in index 0235cb0..aa855e2 100644 --- a/version_config.h.in +++ b/version_config.h.in @@ -1,13 +1,11 @@ #ifndef ICF_VERSION_CONFIG_H #define ICF_VERSION_CONFIG_H -// define your version_libinterface -#define ICF_VERSION_LIB @ICF_VERSION_LIB@ +#define ICF_VERSION_LIB @ICF_VERSION@ -// alternatively you could add your global method getLibInterfaceVersion here const char* getVersion() { - return "@ICF_VERSION_LIB@"; + return "@ICF_VERSION@"; } #endif // ICF_VERSION_CONFIG_H \ No newline at end of file From 0c835934e0476d0ce2bd2e85b77fe85655c8ac34 Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 6 Mar 2020 17:21:26 +0100 Subject: [PATCH 20/22] Undo changes to archive and propagate the version properly to setup.py --- include/icf/archive.h | 38 ++++++++++++++++++++------------------ setup.py.in | 2 +- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/include/icf/archive.h b/include/icf/archive.h index cb5def9..f24b6fb 100644 --- a/include/icf/archive.h +++ b/include/icf/archive.h @@ -247,24 +247,26 @@ class Archive SERIALIZER_FOR_POD(float) SERIALIZER_FOR_POD(double) - Archive& operator&(len_t & v) - { - char first_byte; - m_stream.read(&first_byte, sizeof(char)); - if(!m_stream) { throw std::runtime_error("malformed data"); } - uint8_t number_of_bytes = (first_byte%4)*2-1; - m_stream.read((char*)&v, sizeof(number_of_bytes)); - v+= first_byte/4; - v = Swap(v); - return *this; - } - - const Archive& operator&(len_t v) const - { - v = Swap(v); - m_stream.write((const char*)&v, sizeof(len_t )); - return *this; - } + // Archive& operator&(len_t & v) + // { + // char first_byte; + // m_stream.read(&first_byte, sizeof(char)); + // if(!m_stream) { throw std::runtime_error("malformed data"); } + // uint8_t number_of_bytes = (first_byte%4)*2-1; + // m_stream.read((char*)&v, sizeof(number_of_bytes)); + // v+= first_byte; + // v = Swap(v); + // return *this; + // } + + // const Archive& operator&(len_t v) const + // { + // auto vt = 4*v; + // vt = Swap(vt); + + // m_stream.write((const char*)&v, sizeof(len_t )); + // return *this; + // } diff --git a/setup.py.in b/setup.py.in index 0644376..5e8e7d7 100644 --- a/setup.py.in +++ b/setup.py.in @@ -9,7 +9,7 @@ install_requires = ["numpy"] setup( name="pyicf", - version="@ICF_VERSION_LIB@", + version="@ICF_VERSION@", description="An indexable container file format.", author="Samuel Flis", author_email="samuel.d.flis@gmail.com", From 1b55af675e977b76e28b8386c2e3f5f344c317d5 Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 6 Mar 2020 17:22:34 +0100 Subject: [PATCH 21/22] Undo last change to archive --- include/icf/archive.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/icf/archive.h b/include/icf/archive.h index f24b6fb..6a2c978 100644 --- a/include/icf/archive.h +++ b/include/icf/archive.h @@ -42,7 +42,7 @@ For more information, please refer to #include #include -typedef len_t uint64_t; +// typedef len_t uint64_t; namespace EndianSwapper { From bdf2319d63a59abd0cb77d82a30b2ddabcfbe2c0 Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 6 Mar 2020 17:57:14 +0100 Subject: [PATCH 22/22] Remove deps from build install --- CMakeLists.txt | 4 ++-- setup.py.in | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index edde1a1..9b86b2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,7 +120,7 @@ string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" VERSION_MINOR "${VERSION}") string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.([0-9]+).*" "\\1" VERSION_PATCH "${VERSION}") string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.[0-9]+(.*)" "\\1" VERSION_SHA1 "${VERSION}") set(ICF_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") - +message(STATUS ICF_VERSION) configure_file( version_config.h.in ${CMAKE_BINARY_DIR}/generated/version_config.h ) include_directories( ${CMAKE_BINARY_DIR}/generated/) @@ -151,7 +151,7 @@ if(Python3_Interpreter_FOUND) endif() - configure_file(setup.py.in "${PROJECT_BINARY_DIR}/setup.py") +configure_file(setup.py.in "${PROJECT_BINARY_DIR}/setup.py") # Place the initialization file in the output directory for the Python diff --git a/setup.py.in b/setup.py.in index 5e8e7d7..fc1352d 100644 --- a/setup.py.in +++ b/setup.py.in @@ -4,8 +4,6 @@ ########################################### from setuptools import setup, find_packages -install_requires = ["numpy"] - setup( name="pyicf", @@ -17,7 +15,6 @@ setup( packages=find_packages(), provides=["icf"], license="GNU Lesser General Public License v3 or later", - install_requires=install_requires, package_data = { 'icf': ['_ext/*'] },