refactor: unify parser entity type and add utf16 tests

Co-authored-by: Vladislav Deryabkin <evermake@users.noreply.github.com>

Author: cursoragent

src/parser.ts

@@ -1,7 +1,8 @@
 import type { TextEntity, TgxElement, TgxElementText } from './types.ts'
 
-export type TelegramMessageEntityType
-  = | 'mention'
+export interface MessageEntity {
+  type:
+    | 'mention'
     | 'hashtag'
     | 'cashtag'
     | 'bot_command'
@@ -20,38 +21,15 @@ export type TelegramMessageEntityType
     | 'text_link'
     | 'text_mention'
     | 'custom_emoji'
-
-interface TelegramMessageEntityBase {
-  type: TelegramMessageEntityType | (string & {})
+    | (string & {})
   offset: number
   length: number
-}
-
-interface TelegramMessageEntityTextLink extends TelegramMessageEntityBase {
-  type: 'text_link'
-  url: string
-}
-
-interface TelegramMessageEntityPre extends TelegramMessageEntityBase {
-  type: 'pre'
+  url?: string
   language?: string
-}
-
-interface TelegramMessageEntityCustomEmoji extends TelegramMessageEntityBase {
-  type: 'custom_emoji'
-  custom_emoji_id: string
-}
-
-interface TelegramMessageEntityFallback extends TelegramMessageEntityBase {
+  custom_emoji_id?: string
   [key: string]: unknown
 }
 
-export type TelegramMessageEntity
-  = | TelegramMessageEntityTextLink
-    | TelegramMessageEntityPre
-    | TelegramMessageEntityCustomEmoji
-    | TelegramMessageEntityFallback
-
 interface ParsedEntity {
   start: number
   end: number
@@ -71,7 +49,7 @@ interface OpenEntity {
  */
 export function parseMessageEntities(
   text: string,
-  entities: readonly TelegramMessageEntity[] = [],
+  entities: readonly MessageEntity[] = [],
 ): TgxElement {
   const parsedEntities = collectValidEntities(text, entities)
   const root: TgxElement[] = []
@@ -120,7 +98,7 @@ function buildBoundaries(entities: readonly ParsedEntity[], textLength: number):
 
 function collectValidEntities(
   text: string,
-  entities: readonly TelegramMessageEntity[],
+  entities: readonly MessageEntity[],
 ): ParsedEntity[] {
   const sorted = [...entities]
     .sort((a, b) => {
@@ -166,7 +144,7 @@ function isValidRange(start: number, end: number, textLength: number): boolean {
 
 function mapEntity(
   text: string,
-  entity: TelegramMessageEntity,
+  entity: MessageEntity,
   start: number,
   end: number,
 ): TextEntity | null {

test/parser.test.ts

@@ -1,7 +1,11 @@
-import type { TelegramMessageEntity } from '../src/parser.ts'
+import type { MessageEntity } from '../src/parser.ts'
 import { describe, expect, it } from 'vitest'
 import { parseMessageEntities } from '../src/parser.ts'
 
+function cu(value: string): number {
+  return value.length
+}
+
 describe('parseMessageEntities', () => {
   it('returns plain fragment when entities are missing', () => {
     expect(parseMessageEntities('hello')).toEqual({
@@ -12,7 +16,7 @@ describe('parseMessageEntities', () => {
 
   it('parses nested entities with stable structure', () => {
     const text = 'hello world'
-    const entities: TelegramMessageEntity[] = [
+    const entities: MessageEntity[] = [
       { type: 'bold', offset: 0, length: 11 },
       { type: 'italic', offset: 6, length: 5 },
     ]
@@ -36,7 +40,7 @@ describe('parseMessageEntities', () => {
 
   it('parses adjacent entities correctly', () => {
     const text = 'abcd'
-    const entities: TelegramMessageEntity[] = [
+    const entities: MessageEntity[] = [
       { type: 'bold', offset: 0, length: 2 },
       { type: 'italic', offset: 2, length: 2 },
     ]
@@ -60,7 +64,7 @@ describe('parseMessageEntities', () => {
 
   it('maps link, codeblock and custom emoji entities', () => {
     const text = 'ab🙂de'
-    const entities: TelegramMessageEntity[] = [
+    const entities: MessageEntity[] = [
       { type: 'text_link', offset: 0, length: 2, url: 'https://example.com' },
       { type: 'custom_emoji', offset: 2, length: 2, custom_emoji_id: '42' },
       { type: 'pre', offset: 4, length: 1, language: 'ts' },
@@ -91,7 +95,7 @@ describe('parseMessageEntities', () => {
 
   it('ignores invalid and crossing entities', () => {
     const text = 'abcdef'
-    const entities: TelegramMessageEntity[] = [
+    const entities: MessageEntity[] = [
       { type: 'bold', offset: -1, length: 1 },
       { type: 'italic', offset: 0, length: 0 },
       { type: 'underline', offset: 0, length: 4 },
@@ -116,4 +120,83 @@ describe('parseMessageEntities', () => {
       ],
     })
   })
+
+  it('handles UTF-16 offsets for surrogate pairs and combining marks', () => {
+    const text = 'A🙂e\u0301Ж'
+    const emojiOffset = cu('A')
+    const emojiLength = cu('🙂')
+    const combiningOffset = cu('A🙂')
+    const combiningLength = cu('e\u0301')
+    const cyrillicOffset = cu('A🙂e\u0301')
+    const cyrillicLength = cu('Ж')
+
+    expect(emojiOffset).toBe(1)
+    expect(emojiLength).toBe(2)
+    expect(combiningOffset).toBe(3)
+    expect(combiningLength).toBe(2)
+    expect(cyrillicOffset).toBe(5)
+    expect(cyrillicLength).toBe(1)
+
+    const entities: MessageEntity[] = [
+      { type: 'bold', offset: emojiOffset, length: emojiLength }, // 🙂
+      { type: 'italic', offset: combiningOffset, length: combiningLength }, // e + combining acute
+      { type: 'underline', offset: cyrillicOffset, length: cyrillicLength }, // Ж
+    ]
+
+    expect(parseMessageEntities(text, entities)).toEqual({
+      type: 'fragment',
+      subelements: [
+        { type: 'plain', value: 'A' },
+        {
+          type: 'text',
+          entity: { type: 'bold' },
+          subelements: [{ type: 'plain', value: '🙂' }],
+        },
+        {
+          type: 'text',
+          entity: { type: 'italic' },
+          subelements: [{ type: 'plain', value: 'e\u0301' }],
+        },
+        {
+          type: 'text',
+          entity: { type: 'underline' },
+          subelements: [{ type: 'plain', value: 'Ж' }],
+        },
+      ],
+    })
+  })
+
+  it('handles UTF-16 offsets for ZWJ emoji sequences', () => {
+    const family = '👨‍👩‍👧‍👦'
+    const text = `x${family}y`
+    const familyOffset = cu('x')
+    const familyLength = cu(family)
+    const tailOffset = cu(`x${family}`)
+
+    expect(familyOffset).toBe(1)
+    expect(familyLength).toBe(11)
+    expect(tailOffset).toBe(12)
+
+    const entities: MessageEntity[] = [
+      { type: 'custom_emoji', offset: familyOffset, length: familyLength, custom_emoji_id: 'family-1' },
+      { type: 'code', offset: tailOffset, length: cu('y') },
+    ]
+
+    expect(parseMessageEntities(text, entities)).toEqual({
+      type: 'fragment',
+      subelements: [
+        { type: 'plain', value: 'x' },
+        {
+          type: 'text',
+          entity: { type: 'custom-emoji', id: 'family-1', alt: family },
+          subelements: [],
+        },
+        {
+          type: 'text',
+          entity: { type: 'code' },
+          subelements: [{ type: 'plain', value: 'y' }],
+        },
+      ],
+    })
+  })
 })