From 7f44c67edfda30e6822fc79e35695409ce656447 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Dec 2025 20:52:27 +0000 Subject: [PATCH 1/2] Add Unicode normalization benchmark Implements UTF-8 string normalization (NFC/NFD) benchmark across C, Python, JavaScript, Ruby, and Hemlock. Tests composing combining character sequences and decomposing precomposed characters. --- .../unicode_normalize/unicode_normalize.c | 193 ++++++++++++++++++ .../unicode_normalize/unicode_normalize.hml | 166 +++++++++++++++ .../unicode_normalize/unicode_normalize.js | 28 +++ .../unicode_normalize/unicode_normalize.py | 29 +++ .../unicode_normalize/unicode_normalize.rb | 24 +++ run.sh | 7 +- 6 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 benchmarks/unicode_normalize/unicode_normalize.c create mode 100644 benchmarks/unicode_normalize/unicode_normalize.hml create mode 100644 benchmarks/unicode_normalize/unicode_normalize.js create mode 100644 benchmarks/unicode_normalize/unicode_normalize.py create mode 100644 benchmarks/unicode_normalize/unicode_normalize.rb diff --git a/benchmarks/unicode_normalize/unicode_normalize.c b/benchmarks/unicode_normalize/unicode_normalize.c new file mode 100644 index 0000000..ea194c5 --- /dev/null +++ b/benchmarks/unicode_normalize/unicode_normalize.c @@ -0,0 +1,193 @@ +// Unicode normalization benchmark +// Performs manual UTF-8 string normalization operations +#include +#include +#include + +// Test strings with combining characters and precomposed forms +// NFD: e + combining acute (U+0065 U+0301) vs NFC: é (U+00E9) +// NFD: n + combining tilde (U+006E U+0303) vs NFC: ñ (U+00F1) +// NFD: a + combining ring above (U+0061 U+030A) vs NFC: å (U+00E5) + +// Common combining character mappings (base + combining -> precomposed) +typedef struct { + unsigned int base; + unsigned int combining; + unsigned int composed; +} CompositionMapping; + +static const CompositionMapping compositions[] = { + {0x0065, 0x0301, 0x00E9}, // e + acute -> é + {0x0045, 0x0301, 0x00C9}, // E + acute -> É + {0x0061, 0x0301, 0x00E1}, // a + acute -> á + {0x0041, 0x0301, 0x00C1}, // A + acute -> Á + {0x006F, 0x0301, 0x00F3}, // o + acute -> ó + {0x004F, 0x0301, 0x00D3}, // O + acute -> Ó + {0x0075, 0x0301, 0x00FA}, // u + acute -> ú + {0x0055, 0x0301, 0x00DA}, // U + acute -> Ú + {0x0069, 0x0301, 0x00ED}, // i + acute -> í + {0x0049, 0x0301, 0x00CD}, // I + acute -> Í + {0x006E, 0x0303, 0x00F1}, // n + tilde -> ñ + {0x004E, 0x0303, 0x00D1}, // N + tilde -> Ñ + {0x0061, 0x030A, 0x00E5}, // a + ring -> å + {0x0041, 0x030A, 0x00C5}, // A + ring -> Å + {0x0063, 0x0327, 0x00E7}, // c + cedilla -> ç + {0x0043, 0x0327, 0x00C7}, // C + cedilla -> Ç + {0x0065, 0x0300, 0x00E8}, // e + grave -> è + {0x0045, 0x0300, 0x00C8}, // E + grave -> È + {0x0061, 0x0300, 0x00E0}, // a + grave -> à + {0x0041, 0x0300, 0x00C0}, // A + grave -> À + {0x006F, 0x0302, 0x00F4}, // o + circumflex -> ô + {0x004F, 0x0302, 0x00D4}, // O + circumflex -> Ô + {0x0075, 0x0308, 0x00FC}, // u + diaeresis -> ü + {0x0055, 0x0308, 0x00DC}, // U + diaeresis -> Ü + {0, 0, 0} +}; + +// Decode UTF-8 codepoint, return bytes consumed +int utf8_decode(const unsigned char *s, unsigned int *cp) { + if (s[0] < 0x80) { + *cp = s[0]; + return 1; + } else if ((s[0] & 0xE0) == 0xC0) { + *cp = ((s[0] & 0x1F) << 6) | (s[1] & 0x3F); + return 2; + } else if ((s[0] & 0xF0) == 0xE0) { + *cp = ((s[0] & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); + return 3; + } else if ((s[0] & 0xF8) == 0xF0) { + *cp = ((s[0] & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F); + return 4; + } + *cp = s[0]; + return 1; +} + +// Encode codepoint to UTF-8, return bytes written +int utf8_encode(unsigned int cp, unsigned char *out) { + if (cp < 0x80) { + out[0] = cp; + return 1; + } else if (cp < 0x800) { + out[0] = 0xC0 | (cp >> 6); + out[1] = 0x80 | (cp & 0x3F); + return 2; + } else if (cp < 0x10000) { + out[0] = 0xE0 | (cp >> 12); + out[1] = 0x80 | ((cp >> 6) & 0x3F); + out[2] = 0x80 | (cp & 0x3F); + return 3; + } else { + out[0] = 0xF0 | (cp >> 18); + out[1] = 0x80 | ((cp >> 12) & 0x3F); + out[2] = 0x80 | ((cp >> 6) & 0x3F); + out[3] = 0x80 | (cp & 0x3F); + return 4; + } +} + +// Find composition for base + combining +unsigned int find_composition(unsigned int base, unsigned int combining) { + for (int i = 0; compositions[i].base != 0; i++) { + if (compositions[i].base == base && compositions[i].combining == combining) { + return compositions[i].composed; + } + } + return 0; +} + +// Find decomposition for precomposed character +int find_decomposition(unsigned int composed, unsigned int *base, unsigned int *combining) { + for (int i = 0; compositions[i].base != 0; i++) { + if (compositions[i].composed == composed) { + *base = compositions[i].base; + *combining = compositions[i].combining; + return 1; + } + } + return 0; +} + +// Simple NFC normalization (compose combining sequences) +size_t normalize_nfc(const char *input, char *output, size_t outsize) { + const unsigned char *in = (const unsigned char *)input; + unsigned char *out = (unsigned char *)output; + size_t out_pos = 0; + size_t in_len = strlen(input); + size_t i = 0; + + while (i < in_len && out_pos < outsize - 4) { + unsigned int cp1, cp2; + int len1 = utf8_decode(in + i, &cp1); + + // Check if next char is a combining character + if (i + len1 < in_len) { + int len2 = utf8_decode(in + i + len1, &cp2); + unsigned int composed = find_composition(cp1, cp2); + if (composed) { + out_pos += utf8_encode(composed, out + out_pos); + i += len1 + len2; + continue; + } + } + + out_pos += utf8_encode(cp1, out + out_pos); + i += len1; + } + + output[out_pos] = '\0'; + return out_pos; +} + +// Simple NFD normalization (decompose precomposed characters) +size_t normalize_nfd(const char *input, char *output, size_t outsize) { + const unsigned char *in = (const unsigned char *)input; + unsigned char *out = (unsigned char *)output; + size_t out_pos = 0; + size_t in_len = strlen(input); + size_t i = 0; + + while (i < in_len && out_pos < outsize - 8) { + unsigned int cp; + int len = utf8_decode(in + i, &cp); + + unsigned int base, combining; + if (find_decomposition(cp, &base, &combining)) { + out_pos += utf8_encode(base, out + out_pos); + out_pos += utf8_encode(combining, out + out_pos); + } else { + out_pos += utf8_encode(cp, out + out_pos); + } + + i += len; + } + + output[out_pos] = '\0'; + return out_pos; +} + +int main(int argc, char *argv[]) { + int iterations = argc > 1 ? atoi(argv[1]) : 100000; + + // Test string with mixed NFD and NFC characters + // Contains: café (with é), naïve, résumé, coöperate, señor + const char *test_nfd = "cafe\xCC\x81 na\xC3\xAFve re\xCC\x81sume\xCC\x81 coo\xCC\x88perate sen\xCC\x83or"; + const char *test_nfc = "caf\xC3\xA9 na\xC3\xAFve r\xC3\xA9sum\xC3\xA9 co\xC3\xB6perate se\xC3\xB1or"; + + char buffer1[1024]; + char buffer2[1024]; + long total_len = 0; + + for (int i = 0; i < iterations; i++) { + // NFC: compose combining sequences + size_t len1 = normalize_nfc(test_nfd, buffer1, sizeof(buffer1)); + + // NFD: decompose precomposed characters + size_t len2 = normalize_nfd(test_nfc, buffer2, sizeof(buffer2)); + + total_len += len1 + len2; + } + + printf("%ld\n", total_len); + return 0; +} diff --git a/benchmarks/unicode_normalize/unicode_normalize.hml b/benchmarks/unicode_normalize/unicode_normalize.hml new file mode 100644 index 0000000..a7294ae --- /dev/null +++ b/benchmarks/unicode_normalize/unicode_normalize.hml @@ -0,0 +1,166 @@ +// Unicode normalization benchmark for Hemlock +// Implements simplified NFC/NFD normalization for common Latin characters + +fn parse_int(s) { + let result = 0; + let idx = 0; + while (idx < s.length) { + let ch = s[idx]; + let code: i32 = ch; + let zero: i32 = '0'; + result = result * 10 + (code - zero); + idx = idx + 1; + } + return result; +} + +// Composition mappings: [base, combining, composed] +let compositions = [ + [0x0065, 0x0301, 0x00E9], // e + acute -> é + [0x0045, 0x0301, 0x00C9], // E + acute -> É + [0x0061, 0x0301, 0x00E1], // a + acute -> á + [0x0041, 0x0301, 0x00C1], // A + acute -> Á + [0x006F, 0x0301, 0x00F3], // o + acute -> ó + [0x004F, 0x0301, 0x00D3], // O + acute -> Ó + [0x0075, 0x0301, 0x00FA], // u + acute -> ú + [0x0055, 0x0301, 0x00DA], // U + acute -> Ú + [0x0069, 0x0301, 0x00ED], // i + acute -> í + [0x0049, 0x0301, 0x00CD], // I + acute -> Í + [0x006E, 0x0303, 0x00F1], // n + tilde -> ñ + [0x004E, 0x0303, 0x00D1], // N + tilde -> Ñ + [0x0061, 0x030A, 0x00E5], // a + ring -> å + [0x0041, 0x030A, 0x00C5], // A + ring -> Å + [0x0063, 0x0327, 0x00E7], // c + cedilla -> ç + [0x0043, 0x0327, 0x00C7], // C + cedilla -> Ç + [0x0065, 0x0300, 0x00E8], // e + grave -> è + [0x0045, 0x0300, 0x00C8], // E + grave -> È + [0x0061, 0x0300, 0x00E0], // a + grave -> à + [0x0041, 0x0300, 0x00C0], // A + grave -> À + [0x006F, 0x0302, 0x00F4], // o + circumflex -> ô + [0x004F, 0x0302, 0x00D4], // O + circumflex -> Ô + [0x0075, 0x0308, 0x00FC], // u + diaeresis -> ü + [0x0055, 0x0308, 0x00DC] // U + diaeresis -> Ü +]; + +// Find composition for base + combining +fn find_composition(base, combining) { + let i = 0; + while (i < compositions.length) { + let mapping = compositions[i]; + if (mapping[0] == base && mapping[1] == combining) { + return mapping[2]; + } + i = i + 1; + } + return 0; +} + +// Find decomposition for precomposed character +fn find_decomposition(composed) { + let i = 0; + while (i < compositions.length) { + let mapping = compositions[i]; + if (mapping[2] == composed) { + return [mapping[0], mapping[1]]; + } + i = i + 1; + } + return nil; +} + +// Get codepoint from string at position +fn get_codepoint(s, idx) { + let ch = s[idx]; + let code: i32 = ch; + return code; +} + +// Check if codepoint is a combining character (simplified range check) +fn is_combining(cp) { + // Combining Diacritical Marks: U+0300 - U+036F + return cp >= 0x0300 && cp <= 0x036F; +} + +// Simple NFC normalization (compose combining sequences) +fn normalize_nfc(input) { + let result = []; + let i = 0; + let len = input.length; + + while (i < len) { + let cp1 = get_codepoint(input, i); + + // Check if next char is a combining character + if (i + 1 < len) { + let cp2 = get_codepoint(input, i + 1); + if (is_combining(cp2)) { + let composed = find_composition(cp1, cp2); + if (composed > 0) { + result.push(composed); + i = i + 2; + } else { + result.push(cp1); + i = i + 1; + } + } else { + result.push(cp1); + i = i + 1; + } + } else { + result.push(cp1); + i = i + 1; + } + } + + return result; +} + +// Simple NFD normalization (decompose precomposed characters) +fn normalize_nfd(input) { + let result = []; + let i = 0; + let len = input.length; + + while (i < len) { + let cp = get_codepoint(input, i); + let decomp = find_decomposition(cp); + + if (decomp != nil) { + result.push(decomp[0]); + result.push(decomp[1]); + } else { + result.push(cp); + } + + i = i + 1; + } + + return result; +} + +// Main benchmark +let iterations = 100000; +if (args.length > 1) { + iterations = parse_int(args[1]); +} + +// Test strings with mixed NFD and NFC characters +// Using escape sequences for Unicode characters +let test_nfd = "cafe" + "\u{0301}" + " naive resume" + "\u{0301}" + " cooperate senor"; +let test_nfc = "caf" + "\u{00E9}" + " naive r" + "\u{00E9}" + "sum" + "\u{00E9}" + " cooperate se" + "\u{00F1}" + "or"; + +let total_len = 0; +let i = 0; + +while (i < iterations) { + // NFC: compose combining sequences + let normalized1 = normalize_nfc(test_nfd); + + // NFD: decompose precomposed characters + let normalized2 = normalize_nfd(test_nfc); + + total_len = total_len + normalized1.length + normalized2.length; + i = i + 1; +} + +print(total_len); diff --git a/benchmarks/unicode_normalize/unicode_normalize.js b/benchmarks/unicode_normalize/unicode_normalize.js new file mode 100644 index 0000000..f3320ab --- /dev/null +++ b/benchmarks/unicode_normalize/unicode_normalize.js @@ -0,0 +1,28 @@ +#!/usr/bin/env node +// Unicode normalization benchmark + +function main() { + const iterations = parseInt(process.argv[2]) || 100000; + + // Test string with mixed NFD and NFC characters + // NFD form: base characters followed by combining characters + const testNfd = "cafe\u0301 na\u00EFve re\u0301sume\u0301 coo\u0308perate sen\u0303or"; + // NFC form: precomposed characters + const testNfc = "caf\u00E9 na\u00EFve r\u00E9sum\u00E9 co\u00F6perate se\u00F1or"; + + let totalLen = 0; + + for (let i = 0; i < iterations; i++) { + // NFC: compose combining sequences + const normalized1 = testNfd.normalize('NFC'); + + // NFD: decompose precomposed characters + const normalized2 = testNfc.normalize('NFD'); + + totalLen += normalized1.length + normalized2.length; + } + + console.log(totalLen); +} + +main(); diff --git a/benchmarks/unicode_normalize/unicode_normalize.py b/benchmarks/unicode_normalize/unicode_normalize.py new file mode 100644 index 0000000..f6faedd --- /dev/null +++ b/benchmarks/unicode_normalize/unicode_normalize.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Unicode normalization benchmark +import sys +import unicodedata + +def main(): + iterations = int(sys.argv[1]) if len(sys.argv) > 1 else 100000 + + # Test string with mixed NFD and NFC characters + # NFD form: base characters followed by combining characters + test_nfd = "cafe\u0301 na\u00EFve re\u0301sume\u0301 coo\u0308perate sen\u0303or" + # NFC form: precomposed characters + test_nfc = "caf\u00E9 na\u00EFve r\u00E9sum\u00E9 co\u00F6perate se\u00F1or" + + total_len = 0 + + for _ in range(iterations): + # NFC: compose combining sequences + normalized1 = unicodedata.normalize('NFC', test_nfd) + + # NFD: decompose precomposed characters + normalized2 = unicodedata.normalize('NFD', test_nfc) + + total_len += len(normalized1) + len(normalized2) + + print(total_len) + +if __name__ == "__main__": + main() diff --git a/benchmarks/unicode_normalize/unicode_normalize.rb b/benchmarks/unicode_normalize/unicode_normalize.rb new file mode 100644 index 0000000..7ad08ce --- /dev/null +++ b/benchmarks/unicode_normalize/unicode_normalize.rb @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# Unicode normalization benchmark + +iterations = ARGV[0] ? ARGV[0].to_i : 100000 + +# Test string with mixed NFD and NFC characters +# NFD form: base characters followed by combining characters +test_nfd = "cafe\u0301 na\u00EFve re\u0301sume\u0301 coo\u0308perate sen\u0303or" +# NFC form: precomposed characters +test_nfc = "caf\u00E9 na\u00EFve r\u00E9sum\u00E9 co\u00F6perate se\u00F1or" + +total_len = 0 + +iterations.times do + # NFC: compose combining sequences + normalized1 = test_nfd.unicode_normalize(:nfc) + + # NFD: decompose precomposed characters + normalized2 = test_nfc.unicode_normalize(:nfd) + + total_len += normalized1.length + normalized2.length +end + +puts total_len diff --git a/run.sh b/run.sh index 06bcb6f..bd2cc75 100755 --- a/run.sh +++ b/run.sh @@ -28,7 +28,7 @@ usage() { echo " --iter N Number of iterations (default: 3)" echo " --help, -h Show this help" echo "" - echo "Benchmarks: fib, array_sum, string_concat, primes_sieve, json_serialize, json_deserialize" + echo "Benchmarks: fib, array_sum, string_concat, primes_sieve, json_serialize, json_deserialize, unicode_normalize" echo " (leave empty to run all)" } @@ -78,6 +78,9 @@ get_input() { json_deserialize) [[ $QUICK_MODE -eq 1 ]] && echo 10000 || echo 100000 ;; + unicode_normalize) + [[ $QUICK_MODE -eq 1 ]] && echo 10000 || echo 100000 + ;; esac } @@ -201,7 +204,7 @@ run_all() { if [[ -n "$BENCHMARK" ]]; then benchmarks="$BENCHMARK" else - benchmarks="fib array_sum string_concat primes_sieve json_serialize json_deserialize" + benchmarks="fib array_sum string_concat primes_sieve json_serialize json_deserialize unicode_normalize" fi local languages="c hemlockc hemlock python javascript ruby" From 0a625a2dc98e46a1000a20b30bdccc3c51150a68 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Dec 2025 21:02:25 +0000 Subject: [PATCH 2/2] Fix hemlockc compatibility in unicode_normalize benchmark Replace Unicode escape sequences (\u{XXXX}) with codepoint arrays for hemlockc compiler compatibility. The normalization logic now operates directly on integer codepoint arrays instead of strings. --- .../unicode_normalize/unicode_normalize.hml | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/benchmarks/unicode_normalize/unicode_normalize.hml b/benchmarks/unicode_normalize/unicode_normalize.hml index a7294ae..eab6765 100644 --- a/benchmarks/unicode_normalize/unicode_normalize.hml +++ b/benchmarks/unicode_normalize/unicode_normalize.hml @@ -68,13 +68,6 @@ fn find_decomposition(composed) { return nil; } -// Get codepoint from string at position -fn get_codepoint(s, idx) { - let ch = s[idx]; - let code: i32 = ch; - return code; -} - // Check if codepoint is a combining character (simplified range check) fn is_combining(cp) { // Combining Diacritical Marks: U+0300 - U+036F @@ -82,17 +75,18 @@ fn is_combining(cp) { } // Simple NFC normalization (compose combining sequences) -fn normalize_nfc(input) { +// Works on codepoint arrays +fn normalize_nfc(codepoints) { let result = []; let i = 0; - let len = input.length; + let len = codepoints.length; while (i < len) { - let cp1 = get_codepoint(input, i); + let cp1 = codepoints[i]; // Check if next char is a combining character if (i + 1 < len) { - let cp2 = get_codepoint(input, i + 1); + let cp2 = codepoints[i + 1]; if (is_combining(cp2)) { let composed = find_composition(cp1, cp2); if (composed > 0) { @@ -116,13 +110,14 @@ fn normalize_nfc(input) { } // Simple NFD normalization (decompose precomposed characters) -fn normalize_nfd(input) { +// Works on codepoint arrays +fn normalize_nfd(codepoints) { let result = []; let i = 0; - let len = input.length; + let len = codepoints.length; while (i < len) { - let cp = get_codepoint(input, i); + let cp = codepoints[i]; let decomp = find_decomposition(cp); if (decomp != nil) { @@ -144,20 +139,34 @@ if (args.length > 1) { iterations = parse_int(args[1]); } -// Test strings with mixed NFD and NFC characters -// Using escape sequences for Unicode characters -let test_nfd = "cafe" + "\u{0301}" + " naive resume" + "\u{0301}" + " cooperate senor"; -let test_nfc = "caf" + "\u{00E9}" + " naive r" + "\u{00E9}" + "sum" + "\u{00E9}" + " cooperate se" + "\u{00F1}" + "or"; +// Test data as codepoint arrays (avoids \u{} escape sequences for hemlockc compatibility) +// NFD-style input: base chars + combining chars as codepoints +let test_nfd_codepoints = [ + 0x0063, 0x0061, 0x0066, 0x0065, 0x0301, // café (e + combining acute) + 0x0020, // space + 0x006E, 0x0061, 0x0069, 0x0076, 0x0065, // naive + 0x0020, // space + 0x0072, 0x0065, 0x0301, 0x0073, 0x0075, 0x006D, 0x0065, 0x0301 // résumé +]; + +// NFC-style input: precomposed characters as codepoints +let test_nfc_codepoints = [ + 0x0063, 0x0061, 0x0066, 0x00E9, // café (precomposed é) + 0x0020, // space + 0x006E, 0x0061, 0x0069, 0x0076, 0x0065, // naive + 0x0020, // space + 0x0072, 0x00E9, 0x0073, 0x0075, 0x006D, 0x00E9 // résumé (precomposed) +]; let total_len = 0; let i = 0; while (i < iterations) { - // NFC: compose combining sequences - let normalized1 = normalize_nfc(test_nfd); + // NFC: compose combining sequences (from NFD-style codepoints) + let normalized1 = normalize_nfc(test_nfd_codepoints); - // NFD: decompose precomposed characters - let normalized2 = normalize_nfd(test_nfc); + // NFD: decompose precomposed characters (from NFC-style codepoints) + let normalized2 = normalize_nfd(test_nfc_codepoints); total_len = total_len + normalized1.length + normalized2.length; i = i + 1;