Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ For platforms without native table support (Telegram, WhatsApp, Slack, Instagram
We support the most common Markdown features used in AI responses: headings, code blocks, lists, links, images, blockquotes, horizontal rules, and text formatting. See the Supported Features section for details.

### What about message length limits?
The `TextChunker` intelligently splits long messages at safe breakpoints (after sentences, paragraphs, or list items) while preserving formatting and avoiding broken markup.
The `TextChunker` intelligently splits long messages at safe breakpoints (after sentences, paragraphs, or list items) while preserving formatting and avoiding broken markup. To land on a clean breakpoint, a chunk may exceed the requested `maxLength` by up to 50 characters; budget for that overshoot when choosing your limit (e.g. pass `maxLength: 4046` if you absolutely must stay under Telegram's 4096-char hard cap).

### Can I use this with any PHP version?
This library requires PHP 8.3 or higher, taking advantage of modern PHP features like match expressions and readonly properties.
Expand Down
23 changes: 20 additions & 3 deletions src/Parser.php
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public function parse(string $markdown): IntermediateRepresentation

$blockquoteResult = $this->blockquoteParser->parseLine($line);
if ($blockquoteResult !== null) {
$ir = $ir->addBlock('blockquote', $blockquoteResult);
$ir = $ir->addBlock('blockquote', $this->processInlineForBlockquote($blockquoteResult));
}

continue;
Expand All @@ -125,7 +125,7 @@ public function parse(string $markdown): IntermediateRepresentation
if ($this->blockquoteParser->isInBlockquote()) {
$blockquoteResult = $this->blockquoteParser->parseLine($line);
if ($blockquoteResult !== null) {
$ir = $ir->addBlock('blockquote', $blockquoteResult);
$ir = $ir->addBlock('blockquote', $this->processInlineForBlockquote($blockquoteResult));
}

continue;
Expand Down Expand Up @@ -174,14 +174,31 @@ public function parse(string $markdown): IntermediateRepresentation
if ($this->blockquoteParser->isInBlockquote()) {
$blockquoteResult = $this->blockquoteParser->finishBlockquote();
if ($blockquoteResult !== null) {
$ir = $ir->addBlock('blockquote', $blockquoteResult);
$ir = $ir->addBlock('blockquote', $this->processInlineForBlockquote($blockquoteResult));
}
$this->blockquoteParser->reset();
}

return $ir;
}

private function processInlineForBlockquote(array $blockquoteResult): array
{
$content = $blockquoteResult['content'] ?? '';

if ($this->options['parse_links']) {
$content = $this->linkParser->parse($content);
}

if ($this->options['parse_styles']) {
$content = $this->styleParser->parse($content);
}

$blockquoteResult['content'] = $content;

return $blockquoteResult;
}

public function withOptions(array $options): self
{
$this->options = array_merge($this->options, $options);
Expand Down
2 changes: 1 addition & 1 deletion src/Parsers/LinkParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,6 @@ private function formatLink(string $text, string $url): string

private function formatImage(string $alt, string $url): string
{
return "!{$alt} ({$url})";
return "{$alt} ({$url})";
}
}
43 changes: 42 additions & 1 deletion src/Renderers/AbstractRenderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,48 @@ public function render(IntermediateRepresentation $ir, ?int $maxLength = null):
return $output;
}

abstract protected function renderBlock(array $block): string;
protected function renderBlock(array $block): string
{
return match ($block['type']) {
'paragraph' => $this->renderParagraph($block['content']),
'header' => $this->renderHeader($block['content'], $block['level'] ?? 1),
'code' => $this->renderCodeBlock($block['content'], $block['lang'] ?? null),
'table' => $this->renderTable($block),
'blockquote' => $this->renderBlockquote($block['content']),
'horizontal_rule' => $this->renderHorizontalRule(),
default => '',
};
}

protected function renderParagraph(string $content): string
{
return $content;
}

protected function renderHeader(string $content, int $level): string
{
return $content;
}

protected function renderCodeBlock(string $content, ?string $lang = null): string
{
return $content;
}

protected function renderTable(array $data): string
{
return '';
}

protected function renderBlockquote(string $content): string
{
return "> {$content}";
}

protected function renderHorizontalRule(): string
{
return '---';
}

protected function escapeText(string $text): string
{
Expand Down
17 changes: 1 addition & 16 deletions src/Renderers/DiscordRenderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,6 @@

class DiscordRenderer extends AbstractRenderer
{
protected function renderBlock(array $block): string
{
return match ($block['type']) {
'paragraph' => $this->renderParagraph($block['content']),
'header' => $this->renderHeader($block['content'], $block['level'] ?? 1),
'code' => $this->renderCodeBlock($block['content'], $block['lang'] ?? null),
'table' => $this->renderTable($block),
'blockquote' => $this->renderBlockquote($block['content']),
'horizontal_rule' => $this->renderHorizontalRule(),
default => '',
};
}

protected function escapeText(string $text): string
{
$text = str_replace('\\', '\\\\', $text);
Expand All @@ -40,8 +27,6 @@ protected function renderParagraph(string $content): string

$content = preg_replace_callback('/__HIGHLIGHT__(.+?)__HIGHLIGHT__/', fn ($m) => "**{$m[1]}**", $content);

$content = preg_replace('/!/', '', $content);

$content = $this->convertLinks($content);

return $content;
Expand Down Expand Up @@ -81,7 +66,7 @@ protected function renderTable(array $data): string

protected function renderBlockquote(string $content): string
{
return "> {$content}";
return '> '.$this->renderParagraph($content);
}

protected function renderHorizontalRule(): string
Expand Down
17 changes: 1 addition & 16 deletions src/Renderers/InstagramRenderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,6 @@ class InstagramRenderer extends AbstractRenderer
{
private const HORIZONTAL_RULE = '━━━━━━━━━━━━━━━━';

protected function renderBlock(array $block): string
{
return match ($block['type']) {
'paragraph' => $this->renderParagraph($block['content']),
'header' => $this->renderHeader($block['content'], $block['level'] ?? 1),
'code' => $this->renderCodeBlock($block['content'], $block['lang'] ?? null),
'table' => $this->renderTable($block),
'blockquote' => $this->renderBlockquote($block['content']),
'horizontal_rule' => $this->renderHorizontalRule(),
default => '',
};
}

protected function renderHeader(string $content, int $level): string
{
return UnicodeStyler::bold($content);
Expand Down Expand Up @@ -70,8 +57,6 @@ protected function renderParagraph(string $content): string
$content
);

$content = preg_replace('/!/', '', $content);

$content = preg_replace('/-\s+\[x\]\s*(.*)/', '✅ $1', $content);
$content = preg_replace('/-\s+\[\s\]\s*(.*)/', '⬜ $1', $content);

Expand Down Expand Up @@ -109,7 +94,7 @@ protected function renderTable(array $data): string

protected function renderBlockquote(string $content): string
{
return "❝ {$content} ❞";
return '❝ '.$this->renderParagraph($content).' ❞';
}

protected function renderHorizontalRule(): string
Expand Down
17 changes: 1 addition & 16 deletions src/Renderers/SlackRenderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,6 @@

class SlackRenderer extends AbstractRenderer
{
protected function renderBlock(array $block): string
{
return match ($block['type']) {
'paragraph' => $this->renderParagraph($block['content']),
'header' => $this->renderHeader($block['content'], $block['level'] ?? 1),
'code' => $this->renderCodeBlock($block['content'], $block['lang'] ?? null),
'table' => $this->renderTable($block),
'blockquote' => $this->renderBlockquote($block['content']),
'horizontal_rule' => $this->renderHorizontalRule(),
default => '',
};
}

protected function escapeText(string $text): string
{
$text = str_replace('&', '&', $text);
Expand All @@ -37,8 +24,6 @@ protected function renderParagraph(string $content): string

$content = preg_replace_callback('/__HIGHLIGHT__(.+?)__HIGHLIGHT__/', fn ($m) => "*{$m[1]}*", $content);

$content = preg_replace('/!/', '', $content);

$content = $this->convertLinks($content);
$content = preg_replace_callback('/~~(.+?)~~/', fn ($m) => "~{$m[1]}~", $content);
$content = preg_replace_callback('/\*\*(.+?)\*\*/', fn ($m) => "*{$m[1]}*", $content);
Expand Down Expand Up @@ -81,7 +66,7 @@ protected function renderTable(array $data): string

protected function renderBlockquote(string $content): string
{
return "> {$content}";
return '> '.$this->renderParagraph($content);
}

protected function renderHorizontalRule(): string
Expand Down
19 changes: 1 addition & 18 deletions src/Renderers/TelegramRenderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,6 @@

class TelegramRenderer extends AbstractRenderer
{
protected function renderBlock(array $block): string
{
return match ($block['type']) {
'paragraph' => $this->renderParagraph($block['content']),
'header' => $this->renderHeader($block['content'], $block['level'] ?? 1),
'code' => $this->renderCodeBlock($block['content'], $block['lang'] ?? null),
'table' => $this->renderTable($block),
'blockquote' => $this->renderBlockquote($block['content']),
'horizontal_rule' => $this->renderHorizontalRule(),
default => '',
};
}

protected function escapeText(string $text): string
{
return htmlspecialchars($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
Expand All @@ -42,8 +29,6 @@ protected function renderParagraph(string $content): string
$content = preg_replace_callback('/_(.+?)_/', fn ($m) => "<i>{$m[1]}</i>", $content);
$content = preg_replace_callback('/`(.+?)`/', fn ($m) => "<code>{$m[1]}</code>", $content);

$content = preg_replace('/!/', '', $content);

return $content;
}

Expand Down Expand Up @@ -81,9 +66,7 @@ protected function renderTable(array $data): string

protected function renderBlockquote(string $content): string
{
$content = $this->escapeText($content);

return "💬 {$content}";
return '💬 '.$this->renderParagraph($content);
}

protected function renderHorizontalRule(): string
Expand Down
17 changes: 1 addition & 16 deletions src/Renderers/WhatsAppRenderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,6 @@

class WhatsAppRenderer extends AbstractRenderer
{
protected function renderBlock(array $block): string
{
return match ($block['type']) {
'paragraph' => $this->renderParagraph($block['content']),
'header' => $this->renderHeader($block['content'], $block['level'] ?? 1),
'code' => $this->renderCodeBlock($block['content'], $block['lang'] ?? null),
'table' => $this->renderTable($block),
'blockquote' => $this->renderBlockquote($block['content']),
'horizontal_rule' => $this->renderHorizontalRule(),
default => '',
};
}

protected function renderHeader(string $content, int $level): string
{
return "*{$content}*";
Expand All @@ -39,8 +26,6 @@ protected function renderParagraph(string $content): string

$content = preg_replace_callback('/(.+?) \((https?:\/\/[^\)]+)\)/', fn ($matches) => "{$matches[1]}: {$matches[2]}", $content);

$content = preg_replace('/!/', '', $content);

$content = preg_replace('/-\s+\[x\]\s*(.*)/', '✅ $1', $content);
$content = preg_replace('/-\s+\[\s\]\s*(.*)/', '⬜ $1', $content);

Expand Down Expand Up @@ -72,7 +57,7 @@ protected function renderTable(array $data): string

protected function renderBlockquote(string $content): string
{
return "💬 {$content}";
return '💬 '.$this->renderParagraph($content);
}

protected function renderHorizontalRule(): string
Expand Down
5 changes: 5 additions & 0 deletions src/Support/TextChunker.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ final class TextChunker
{
private const SAFE_BREAKPOINTS = [' ', "\n", "\t", '.', ',', ';', ':', '-', '!', '?', ')', ']', '}', '"', "'"];

/**
* Maximum number of characters a chunk is allowed to exceed `maxLength` by
* in order to land on a safe breakpoint instead of mid-word. Callers that
* must respect a hard cap should pass `maxLength = hardCap - 50`.
*/
private const MAX_CHUNK_OVERSHOOT = 50;

public static function chunk(string $text, int $maxLength, string $encoding = 'UTF-8'): array
Expand Down
Loading