diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 66b5cfe96..084544d62 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -51,12 +51,14 @@ jobs: with: key: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.flavor }}-python - - name: Skip manylinux for musllinux target + - name: Setup musllinux build if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }} + # workaround: set CXX to g++, so it does not pick clang++ for python 3.14 builds run: | echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV + echo "CIBW_ENVIRONMENT_LINUX=CXX=g++" >> $GITHUB_ENV - - name: Skip musllinux for manylinux target + - name: Setup manylinux build if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }} run: | echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV @@ -101,15 +103,15 @@ jobs: # skip tests on pypy, currently fails for indexer tests CIBW_TEST_SKIP: "pp*" - # (many)linux custom docker images + # (many,musl)linux custom docker images CIBW_MANYLINUX_X86_64_IMAGE: 'keyvidev/manylinux-builder-x86_64' CIBW_MANYLINUX_AARCH64_IMAGE: 'keyvidev/manylinux-builder-aarch64' CIBW_MUSLLINUX_X86_64_IMAGE: 'keyvidev/musllinux-builder-x86_64' CIBW_MUSLLINUX_AARCH64_IMAGE: 'keyvidev/musllinux-builder-aarch64' - # ccache using path - CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH - CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf + # prefix ccache in path + CIBW_ENVIRONMENT_MACOS: ${{ env.CIBW_ENVIRONMENT_MACOS }} PATH=/usr/local/opt/ccache/libexec:/opt/homebrew/opt/ccache/libexec:$PATH + CIBW_ENVIRONMENT_LINUX: ${{ env.CIBW_ENVIRONMENT_LINUX }} PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf # python dependencies CIBW_BEFORE_BUILD: pip install -r python/requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index bf3a5fd12..ee9148166 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,9 +94,9 @@ if(Boost_VERSION VERSION_GREATER_EQUAL "1.86.0") endif() if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(_KEYVI_LINK_LIBRARIES_STATIC "${_KEYVI_LINK_LIBRARIES_STATIC} boost_program_options boost_iostreams boost_filesystem boost_regex boost_thread") + set(_KEYVI_LINK_LIBRARIES_STATIC "${_KEYVI_LINK_LIBRARIES_STATIC} boost_program_options boost_iostreams boost_filesystem") else () - set(_KEYVI_LINK_LIBRARIES_DYNAMIC "${_KEYVI_LINK_LIBRARIES_DYNAMIC} boost_program_options boost_iostreams boost_filesystem boost_regex boost_thread") + set(_KEYVI_LINK_LIBRARIES_DYNAMIC "${_KEYVI_LINK_LIBRARIES_DYNAMIC} boost_program_options boost_iostreams boost_filesystem") endif () if(Boost_VERSION VERSION_GREATER_EQUAL "1.86.0") diff --git a/README.md b/README.md index f369fc080..a7390363d 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![PythonImpl](https://img.shields.io/pypi/implementation/keyvi.svg)](https://pypi.python.org/pypi/keyvi/) [![PythonFormat](https://img.shields.io/pypi/format/keyvi.svg)](https://pypi.python.org/pypi/keyvi/) [![PyPIVersion](https://img.shields.io/pypi/v/keyvi.svg)](https://pypi.python.org/pypi/keyvi/) +[![CratesVersion](https://img.shields.io/crates/v/keyvi)](https://crates.io/crates/keyvi/) ## ![Keyvi](/doc/images/keyvi-small.png) @@ -42,10 +43,10 @@ The python extension can be compiled standalone, check the [Readme](/python/READ * [Index](/doc/usage/Keyvi%20Index%20with%20python.md) * [Crashcourse](/doc/usage/Crashcourse.md) * [API docs](https://keyvidev.github.io/keyvi/index.html) - * [Using python keyvi with EMR (mrjob or pyspark)](/doc/usage/Using%20pykeyvi%20in%20EMR.md) + * [Using python keyvi with EMR (mrjob or pyspark)](/doc/usage/Using%20pykeyvi%20in%20EMR.md) ## Internals - + * [Construction Basics](/doc/algorithm/Construction-Basics.md) * [Persistence Basics](/doc/algorithm/Persistence-Basics.md) * [Minimization](/doc/algorithm/Minimization.md) @@ -59,7 +60,7 @@ If you like to go deep down in the basics, keyvi is inspired by the following 2 ## Release procedure * [How to make a release](doc/RELEASE_PROCESS.md) - + ## License keyvi is licensed under Apache License 2.0("ALv2"), see [license](LICENSE) for details, all [3rdparty libraries](/keyvi/3rdparty) ship with their own license. Except Boost, Snappy and zlib all 3rdparty code can be exclusively found in the [3rdparty](/keyvi/3rdparty) folder. The following licenses are used for the 3rdparty code (last updated: `0.5.0`, provided without warranty). diff --git a/python/setup.py b/python/setup.py index f9ca78dd5..03a0bb890 100644 --- a/python/setup.py +++ b/python/setup.py @@ -28,7 +28,7 @@ try: cpu_count = multiprocessing.cpu_count() -except: +except NotImplementedError: cpu_count = 1 ################# @@ -82,9 +82,13 @@ def generate_pykeyvi_source(): pxds = glob.glob("src/pxds/*") converters = "src/converters" converter_files = glob.glob(path.join(converters, "*")) - max_modification_time = max([path.getmtime(fn) for fn in addons + pxds + converter_files]) + max_modification_time = max( + [path.getmtime(fn) for fn in addons + pxds + converter_files] + ) - if not path.exists(pykeyvi_cpp) or max_modification_time > path.getmtime(pykeyvi_cpp): + if not path.exists(pykeyvi_cpp) or max_modification_time > path.getmtime( + pykeyvi_cpp + ): try: import autowrap.Main @@ -101,7 +105,9 @@ def generate_pykeyvi_source(): if not path.exists(pykeyvi_cpp): raise else: - print("Could not find autowrap, probably running from sdist environment") + print( + "Could not find autowrap, probably running from sdist environment" + ) @contextmanager @@ -113,7 +119,9 @@ def symlink_keyvi(): os.symlink(path.abspath(keyvi_cpp_source), keyvi_cpp_link) shutil.copy("../CMakeLists.txt", path.join(keyvi_cpp, "CMakeLists.txt")) shutil.copytree("../cmake_modules", path.join(keyvi_cpp, "cmake_modules")) - keyvi_source_path = os.path.realpath(os.path.join(os.getcwd(), keyvi_cpp_source)) + keyvi_source_path = os.path.realpath( + os.path.join(os.getcwd(), keyvi_cpp_source) + ) pykeyvi_source_path = os.path.join(os.getcwd(), keyvi_cpp_link) yield (pykeyvi_source_path, keyvi_source_path) finally: @@ -138,7 +146,7 @@ def cmake_configure( cmake_configure_cmd = "mkdir -p {}".format(build_path) cmake_configure_cmd += " && cd {}".format(build_path) - cmake_configure_cmd += " && cmake" ' -D CMAKE_CXX_FLAGS="{CXX_FLAGS}"'.format( + cmake_configure_cmd += ' && cmake -D CMAKE_CXX_FLAGS="{CXX_FLAGS}"'.format( CXX_FLAGS=CMAKE_CXX_FLAGS.strip() ) cmake_configure_cmd += f" -D CMAKE_BUILD_TYPE={build_type}" @@ -167,7 +175,9 @@ def cmake_configure( cmake_flags[k] = " ".join(v.split()) # set additional compiler flags - set_additional_flags("extra_compile_args", cmake_flags["KEYVI_CXX_FLAGS_ALL"].split(" ")) + set_additional_flags( + "extra_compile_args", cmake_flags["KEYVI_CXX_FLAGS_ALL"].split(" ") + ) # set defines if cmake_flags["KEYVI_COMPILE_DEFINITIONS"]: @@ -188,7 +198,9 @@ def cmake_configure( # set link libraries if cmake_flags["KEYVI_LINK_LIBRARIES_STATIC"]: if sys.platform == "darwin": - set_additional_flags("libraries", cmake_flags["KEYVI_LINK_LIBRARIES_STATIC"].split(" ")) + set_additional_flags( + "libraries", cmake_flags["KEYVI_LINK_LIBRARIES_STATIC"].split(" ") + ) else: extra_link_arguments = ["-Wl,-Bstatic"] @@ -200,11 +212,15 @@ def cmake_configure( set_additional_flags("extra_link_args", extra_link_arguments) if cmake_flags["KEYVI_LINK_LIBRARIES_DYNAMIC"]: - set_additional_flags("libraries", cmake_flags["KEYVI_LINK_LIBRARIES_DYNAMIC"].split(" ")) + set_additional_flags( + "libraries", cmake_flags["KEYVI_LINK_LIBRARIES_DYNAMIC"].split(" ") + ) # set link args if cmake_flags["KEYVI_LINK_FLAGS"]: - set_additional_flags("extra_link_args", cmake_flags["KEYVI_LINK_FLAGS"].split(" ")) + set_additional_flags( + "extra_link_args", cmake_flags["KEYVI_LINK_FLAGS"].split(" ") + ) return cmake_flags @@ -218,7 +234,9 @@ def set_additional_flags(key, additional_flags): def patch_for_custom_zlib(zlib_root): for ext_m in ext_modules: - include_dirs = [path.join(zlib_root, "include")] + getattr(ext_m, "include_dirs") + include_dirs = [path.join(zlib_root, "include")] + getattr( + ext_m, "include_dirs" + ) setattr(ext_m, "include_dirs", include_dirs) library_dirs = [path.join(zlib_root, "lib")] + getattr(ext_m, "library_dirs") setattr(ext_m, "library_dirs", library_dirs) @@ -275,7 +293,7 @@ def load_options(self): f = open(path.join(keyvi_build_dir, "custom_opts"), "r") self.options = json.loads(f.readline()) return - except: + except FileNotFoundError: pass self.options["mode"] = "release" if not self.mode else self.mode if self.zlib_root: @@ -293,7 +311,9 @@ def run(self): osx_architectures = None archflags = os.environ.get("ARCHFLAGS") if archflags is not None: - osx_architectures = ";".join(set(archflags.split()) & {"x86_64", "arm64"}) + osx_architectures = ";".join( + set(archflags.split()) & {"x86_64", "arm64"} + ) cmake_prefix_path = os.environ.get("_CMAKE_PREFIX_PATH") @@ -340,7 +360,9 @@ def run(self): if "zlib_root" in self.options: patch_for_custom_zlib(self.options["zlib_root"]) - keyvi_build_cmd = "cd {} && make -j {} bindings".format(keyvi_build_dir, cpu_count) + keyvi_build_cmd = "cd {} && make -j {} bindings".format( + keyvi_build_dir, cpu_count + ) print("Building keyvi C++ part: " + keyvi_build_cmd) subprocess.call(keyvi_build_cmd, shell=True) @@ -418,6 +440,7 @@ class build_ext(custom_opts, build_cxx): "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: MacOS :: MacOS X",