From 44e5da7c0ae0bb4579a54bbdca3cd21db8307960 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 22 Jan 2026 15:55:08 +0100 Subject: [PATCH 01/13] explicit c++20 --- src/Makevars | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Makevars b/src/Makevars index 77229ae..b138adb 100644 --- a/src/Makevars +++ b/src/Makevars @@ -1 +1,4 @@ -PKG_CPPFLAGS = -DSTRICT_R_HEADERS \ No newline at end of file +PKG_CPPFLAGS = -DSTRICT_R_HEADERS +PKG_CXXFLAGS = -std=gnu++20 +CXX = g++ -std=gnu++20 +CXX_STD=CXX20 From a8adc83b7b838c2a391e1b5ae3eeb675221be9ba Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 22 Jan 2026 16:05:52 +0100 Subject: [PATCH 02/13] reorder persistent_unordered_map_fnv --- src/udpipe.cpp | 77 +++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index 5320ef2..e2cdcbf 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3471,6 +3471,44 @@ T* unaligned_upper_bound(T* first, size_t size, T val) { namespace morphodita { +// Definitions - Move fnv_hash BEFORE persistent_unordered_map +struct persistent_unordered_map_fnv_hash { + persistent_unordered_map_fnv_hash(unsigned num) { + mask = 1; + while (mask < num) + mask <<= 1; + hash.resize(mask + 1); + mask--; + } + persistent_unordered_map_fnv_hash(binary_decoder& data) { + uint32_t size = data.next_4B(); + mask = size - 2; + hash.resize(size); + memcpy(hash.data(), (const void*)data.next(size), size * sizeof(uint32_t)); + + size = data.next_4B(); + this->data.resize(size); + if (size) memcpy(this->data.data(), data.next(size), size); + } + + inline uint32_t index(const char* data, int len) const { + if (len <= 0) return 0; + if (len == 1) return unaligned_load(data); + if (len == 2) return unaligned_load(data); + + uint32_t hash = 2166136261U; + while (len--) + hash = (hash ^ unsigned(*data++)) * 16777619U; + return hash & mask; + } + + inline void save(binary_encoder& enc); + + unsigned mask; + vector hash; + vector data; +}; + // Declarations class persistent_unordered_map { public: @@ -3510,50 +3548,13 @@ class persistent_unordered_map { inline void save(binary_encoder& enc); private: - struct fnv_hash; + using fnv_hash = persistent_unordered_map_fnv_hash; vector hashes; template void construct(const map& map, double load_factor, EntryEncode entry_encode); }; -// Definitions -struct persistent_unordered_map::fnv_hash { - fnv_hash(unsigned num) { - mask = 1; - while (mask < num) - mask <<= 1; - hash.resize(mask + 1); - mask--; - } - fnv_hash(binary_decoder& data) { - uint32_t size = data.next_4B(); - mask = size - 2; - hash.resize(size); - memcpy(hash.data(), (const void*)data.next(size), size * sizeof(uint32_t)); - - size = data.next_4B(); - this->data.resize(size); - if (size) memcpy(this->data.data(), data.next(size), size); - } - - inline uint32_t index(const char* data, int len) const { - if (len <= 0) return 0; - if (len == 1) return unaligned_load(data); - if (len == 2) return unaligned_load(data); - - uint32_t hash = 2166136261U; - while (len--) - hash = (hash ^ unsigned(*data++)) * 16777619U; - return hash & mask; - } - - inline void save(binary_encoder& enc); - - unsigned mask; - vector hash; - vector data; -}; template const unsigned char* persistent_unordered_map::at(const char* str, int len, EntrySize entry_size) const { From d9b5943ea3e6483bb774a523fc0378ef72914d93 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 22 Jan 2026 16:12:00 +0100 Subject: [PATCH 03/13] back --- src/udpipe.cpp | 79 +++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index e2cdcbf..bf8ebf4 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3471,44 +3471,6 @@ T* unaligned_upper_bound(T* first, size_t size, T val) { namespace morphodita { -// Definitions - Move fnv_hash BEFORE persistent_unordered_map -struct persistent_unordered_map_fnv_hash { - persistent_unordered_map_fnv_hash(unsigned num) { - mask = 1; - while (mask < num) - mask <<= 1; - hash.resize(mask + 1); - mask--; - } - persistent_unordered_map_fnv_hash(binary_decoder& data) { - uint32_t size = data.next_4B(); - mask = size - 2; - hash.resize(size); - memcpy(hash.data(), (const void*)data.next(size), size * sizeof(uint32_t)); - - size = data.next_4B(); - this->data.resize(size); - if (size) memcpy(this->data.data(), data.next(size), size); - } - - inline uint32_t index(const char* data, int len) const { - if (len <= 0) return 0; - if (len == 1) return unaligned_load(data); - if (len == 2) return unaligned_load(data); - - uint32_t hash = 2166136261U; - while (len--) - hash = (hash ^ unsigned(*data++)) * 16777619U; - return hash & mask; - } - - inline void save(binary_encoder& enc); - - unsigned mask; - vector hash; - vector data; -}; - // Declarations class persistent_unordered_map { public: @@ -3548,7 +3510,7 @@ class persistent_unordered_map { inline void save(binary_encoder& enc); private: - using fnv_hash = persistent_unordered_map_fnv_hash; + struct fnv_hash; vector hashes; template @@ -3556,6 +3518,45 @@ class persistent_unordered_map { }; +// Definitions +struct persistent_unordered_map::fnv_hash { + fnv_hash(unsigned num) { + mask = 1; + while (mask < num) + mask <<= 1; + hash.resize(mask + 1); + mask--; + } + fnv_hash(binary_decoder& data) { + uint32_t size = data.next_4B(); + mask = size - 2; + hash.resize(size); + memcpy(hash.data(), (const void*)data.next(size), size * sizeof(uint32_t)); + + size = data.next_4B(); + this->data.resize(size); + if (size) memcpy(this->data.data(), data.next(size), size); + } + + inline uint32_t index(const char* data, int len) const { + if (len <= 0) return 0; + if (len == 1) return unaligned_load(data); + if (len == 2) return unaligned_load(data); + + uint32_t hash = 2166136261U; + while (len--) + hash = (hash ^ unsigned((signed char)*data++)) * 16777619U; + return hash & mask; + } + + inline void save(binary_encoder& enc); + + unsigned mask; + vector hash; + vector data; +}; + + template const unsigned char* persistent_unordered_map::at(const char* str, int len, EntrySize entry_size) const { if (unsigned(len) >= hashes.size()) return nullptr; From 4926f1b71820d07acc81a5cec3887ec618b74df1 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 22 Jan 2026 16:16:55 +0100 Subject: [PATCH 04/13] makevars --- src/Makevars | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Makevars b/src/Makevars index b138adb..270f5e9 100644 --- a/src/Makevars +++ b/src/Makevars @@ -1,4 +1,2 @@ PKG_CPPFLAGS = -DSTRICT_R_HEADERS -PKG_CXXFLAGS = -std=gnu++20 CXX = g++ -std=gnu++20 -CXX_STD=CXX20 From 9c9c4465538933cf5ebbb129ac297ab6d751ff50 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 22 Jan 2026 16:21:04 +0100 Subject: [PATCH 05/13] hop --- src/udpipe.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index bf8ebf4..be7a98f 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3545,7 +3545,7 @@ struct persistent_unordered_map::fnv_hash { uint32_t hash = 2166136261U; while (len--) - hash = (hash ^ unsigned((signed char)*data++)) * 16777619U; + hash = (hash ^ unsigned(*data++)) * 16777619U; return hash & mask; } From 34f90cc144b106e2d75a358261c69a5c2aa8f90d Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 22 Jan 2026 16:27:57 +0100 Subject: [PATCH 06/13] makevars --- src/Makevars | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makevars b/src/Makevars index 270f5e9..d783e60 100644 --- a/src/Makevars +++ b/src/Makevars @@ -1,2 +1,3 @@ PKG_CPPFLAGS = -DSTRICT_R_HEADERS CXX = g++ -std=gnu++20 +CXX_STD = CXX20 From b6c752e5791ce71a706bb3da4e38b47de72fa3e8 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 29 Jan 2026 09:35:27 +0100 Subject: [PATCH 07/13] inline --- src/udpipe.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index be7a98f..a0513a9 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3492,7 +3492,7 @@ class persistent_unordered_map { inline const unsigned char* data_start(int len) const; // Creation functions - persistent_unordered_map() {} + inline persistent_unordered_map() {} template persistent_unordered_map(const unordered_map& map, double load_factor, EntryEncode entry_encode); template From aea561a8f7aa07599d04fe440f2130ca0f5164a4 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 29 Jan 2026 09:41:47 +0100 Subject: [PATCH 08/13] persistent_unordered_map::persistent_unordered_map() {} --- src/udpipe.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index a0513a9..b07f23c 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3636,6 +3636,8 @@ const unsigned char* persistent_unordered_map::data_start(int len) const { return unsigned(len) < hashes.size() ? hashes[len].data.data() : nullptr; } +persistent_unordered_map::persistent_unordered_map() {} + void persistent_unordered_map::resize(unsigned elems) { if (hashes.size() == 0) hashes.emplace_back(1); else if (hashes.size() == 1) hashes.emplace_back(1<<8); From 0a4b4ed76b04aa28778c58aa2b45afe25cca5592 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Thu, 29 Jan 2026 09:55:14 +0100 Subject: [PATCH 09/13] inline persistent_unordered_map(); --- src/udpipe.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index b07f23c..bc5da95 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3492,7 +3492,7 @@ class persistent_unordered_map { inline const unsigned char* data_start(int len) const; // Creation functions - inline persistent_unordered_map() {} + inline persistent_unordered_map(); template persistent_unordered_map(const unordered_map& map, double load_factor, EntryEncode entry_encode); template From 4ca04e56193d68ceacacfc91336dc84d3355905f Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Fri, 30 Jan 2026 09:09:48 +0100 Subject: [PATCH 10/13] spaces --- src/udpipe.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/udpipe.cpp b/src/udpipe.cpp index bc5da95..9664062 100644 --- a/src/udpipe.cpp +++ b/src/udpipe.cpp @@ -3517,7 +3517,6 @@ class persistent_unordered_map { void construct(const map& map, double load_factor, EntryEncode entry_encode); }; - // Definitions struct persistent_unordered_map::fnv_hash { fnv_hash(unsigned num) { @@ -3556,7 +3555,6 @@ struct persistent_unordered_map::fnv_hash { vector data; }; - template const unsigned char* persistent_unordered_map::at(const char* str, int len, EntrySize entry_size) const { if (unsigned(len) >= hashes.size()) return nullptr; From 8f4648db0b3342899eb703fda9d55d5bddbf39a4 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Fri, 30 Jan 2026 09:34:35 +0100 Subject: [PATCH 11/13] disable explicit C++20 --- src/Makevars | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Makevars b/src/Makevars index d783e60..62b2507 100644 --- a/src/Makevars +++ b/src/Makevars @@ -1,3 +1 @@ PKG_CPPFLAGS = -DSTRICT_R_HEADERS -CXX = g++ -std=gnu++20 -CXX_STD = CXX20 From 2222c9a314140294bbe320872f97052d5563c725 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Fri, 30 Jan 2026 09:34:43 +0100 Subject: [PATCH 12/13] news and bump --- DESCRIPTION | 2 +- NEWS.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 38c3e3f..8f7005c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: udpipe Type: Package Title: Tokenization, Parts of Speech Tagging, Lemmatization and Dependency Parsing with the 'UDPipe' 'NLP' Toolkit -Version: 0.8.15 +Version: 0.8.16 Maintainer: Jan Wijffels Authors@R: c( person('Jan', 'Wijffels', role = c('aut', 'cre', 'cph'), email = 'jwijffels@bnosac.be', comment = 'R wrapper'), diff --git a/NEWS.md b/NEWS.md index f704e99..e9b34fa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +## CHANGES IN udpipe VERSION 0.8.16 + +- Fix on the declaration of persistent_unordered_map for C++20 + ## CHANGES IN udpipe VERSION 0.8.15 - Drop C++11 from Makevars From 17857fc158889d3e9a1fa58fc30ca44e0bd727e2 Mon Sep 17 00:00:00 2001 From: Jan Wijffels Date: Fri, 30 Jan 2026 09:45:06 +0100 Subject: [PATCH 13/13] newline --- src/Makevars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makevars b/src/Makevars index 62b2507..77229ae 100644 --- a/src/Makevars +++ b/src/Makevars @@ -1 +1 @@ -PKG_CPPFLAGS = -DSTRICT_R_HEADERS +PKG_CPPFLAGS = -DSTRICT_R_HEADERS \ No newline at end of file