Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,10 @@ frosh_tools:
Plugin system config (Admin → Extensions → Frosh Tools):

- **Monitor mail address** — recipient for `frosh:monitor`
- **Queue grace time** (minutes) — when a queue is considered stuck
- **Default queue grace time** (minutes) — when a pending message is considered stuck
- **Exclude failed queues** — ignore transports whose name contains `failed` (default on)
- **Queues to monitor** — optional allowlist, e.g. `async, low_priority`
- **Per-queue grace times** — optional overrides, e.g. `async:15, low_priority:60`
- **Task grace time** (minutes) — when a scheduled task is considered stuck

JSON Schema for IDE validation: [`frosh-tools-schema.json`](frosh-tools-schema.json).
Expand Down
202 changes: 186 additions & 16 deletions src/Components/Health/Checker/HealthChecker/QueueChecker.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace Frosh\Tools\Components\Health\Checker\HealthChecker;

use Doctrine\DBAL\ArrayParameterType;
use Doctrine\DBAL\Connection;
use Frosh\Tools\Components\Health\Checker\CheckerInterface;
use Frosh\Tools\Components\Health\HealthCollection;
Expand All @@ -12,6 +13,12 @@

class QueueChecker implements HealthCheckerInterface, CheckerInterface
{
private const CONFIG_GRACE = 'FroshTools.config.monitorQueueGraceTime';
private const CONFIG_EXCLUDE_FAILED = 'FroshTools.config.monitorExcludeFailedQueues';
private const CONFIG_QUEUES = 'FroshTools.config.monitorQueues';
private const CONFIG_GRACE_TIMES = 'FroshTools.config.monitorQueueGraceTimes';
private const DEFAULT_GRACE_MINUTES = 15;

public function __construct(
private readonly Connection $connection,
private readonly SystemConfigService $configService,
Expand All @@ -20,29 +27,192 @@ public function __construct(

public function collect(HealthCollection $collection): void
{
$maxDiff = $this->configService->getInt('FroshTools.config.monitorQueueGraceTime') ?: 15;
$oldMessageLimit = (new \DateTimeImmutable())->modify(\sprintf('-%d minutes', $maxDiff));
$defaultGrace = $this->configService->getInt(self::CONFIG_GRACE) ?: self::DEFAULT_GRACE_MINUTES;
$excludeFailed = $this->shouldExcludeFailedQueues();
$queues = $this->parseCsvList($this->configService->getString(self::CONFIG_QUEUES));
$graceByQueue = $this->parseGraceMap($this->configService->getString(self::CONFIG_GRACE_TIMES));

$snippet = 'Open Queues';
$recommended = \sprintf('max %d mins', $maxDiff);

/** @var string|false $oldestMessageAt */
$oldestMessageAt = $this->connection->fetchOne('SELECT available_at FROM messenger_messages WHERE available_at < UTC_TIMESTAMP() ORDER BY available_at ASC LIMIT 1');
$pendingByQueue = $this->fetchOldestPendingMessagePerQueue($excludeFailed, $queues);

if (\is_string($oldestMessageAt)) {
$diff = round(abs(
((new \DateTime($oldestMessageAt . ' UTC'))->getTimestamp() - $oldMessageLimit->getTimestamp()) / 60,
if ($pendingByQueue === []) {
$collection->add(SettingsResult::info(
'queue',
$snippet,
'0 mins',
\sprintf('max %d mins', $defaultGrace),
));

if ($diff > $maxDiff) {
$result = SettingsResult::warning('queue', $snippet, $diff . ' mins', $recommended);
} else {
$result = SettingsResult::ok('queue', $snippet, $diff . ' mins', $recommended);
return;
}

$worst = $this->selectWorstQueue($pendingByQueue, $graceByQueue, $defaultGrace);
$recommended = \sprintf('max %d mins', $worst['grace']);
$current = \sprintf('%d mins (%s)', $worst['ageMinutes'], $worst['queueName']);

if ($worst['overdue']) {
$collection->add(SettingsResult::warning('queue', $snippet, $current, $recommended));

return;
}

$collection->add(SettingsResult::ok('queue', $snippet, $current, $recommended));
}

/**
* @param list<string> $queues
*
* @return list<array{available_at: string, queue_name: string}>
*/
private function fetchOldestPendingMessagePerQueue(bool $excludeFailed, array $queues): array
{
// One row per queue (oldest pending message). Evaluating each queue against its
// own grace avoids masking a tighter queue behind an older message on a looser one.
$query = $this->connection->createQueryBuilder()
->select('queue_name', 'MIN(available_at) AS available_at')
->from('messenger_messages')
->where('available_at <= UTC_TIMESTAMP()')
->groupBy('queue_name')
->orderBy('available_at', 'ASC');

if ($excludeFailed) {
// Symfony failure transport names typically contain "failed" (e.g. async_failed).
$query
->andWhere('queue_name NOT LIKE :failedPattern')
->setParameter('failedPattern', '%failed%');
}

if ($queues !== []) {
$query
->andWhere('queue_name IN (:queues)')
->setParameter('queues', $queues, ArrayParameterType::STRING);
}

/** @var list<array{available_at: string, queue_name: string}> $rows */
$rows = $query->fetchAllAssociative();

return $rows;
}

/**
* Prefer any overdue queue (highest minutes-over-grace); otherwise the oldest pending age.
*
* @param list<array{available_at: string, queue_name: string}> $pendingByQueue
* @param array<string, int> $graceByQueue
*
* @return array{queueName: string, ageMinutes: int, grace: int, overdue: bool}
*/
private function selectWorstQueue(array $pendingByQueue, array $graceByQueue, int $defaultGrace): array
{
$worst = null;

foreach ($pendingByQueue as $row) {
$queueName = (string) $row['queue_name'];
$grace = $graceByQueue[$queueName] ?? $defaultGrace;
$ageMinutes = $this->ageInMinutes((string) $row['available_at']);
$overdue = $ageMinutes > $grace;
$overBy = $overdue ? $ageMinutes - $grace : 0;

$candidate = [
'queueName' => $queueName,
'ageMinutes' => $ageMinutes,
'grace' => $grace,
'overdue' => $overdue,
'overBy' => $overBy,
];

if ($worst === null) {
$worst = $candidate;
continue;
}

// Overdue always beats healthy.
if ($candidate['overdue'] !== $worst['overdue']) {
if ($candidate['overdue']) {
$worst = $candidate;
}
continue;
}

// Same state: both overdue → furthest past grace; both healthy → oldest age.
if ($candidate['overdue']) {
if ($candidate['overBy'] > $worst['overBy']
|| ($candidate['overBy'] === $worst['overBy'] && $candidate['ageMinutes'] > $worst['ageMinutes'])) {
$worst = $candidate;
}
continue;
}

if ($candidate['ageMinutes'] > $worst['ageMinutes']) {
$worst = $candidate;
}
}

\assert($worst !== null);

return [
'queueName' => $worst['queueName'],
'ageMinutes' => $worst['ageMinutes'],
'grace' => $worst['grace'],
'overdue' => $worst['overdue'],
];
}

private function ageInMinutes(string $availableAt): int
{
$available = new \DateTimeImmutable($availableAt, new \DateTimeZone('UTC'));
$now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'));
$seconds = \max(0, $now->getTimestamp() - $available->getTimestamp());

return (int) \floor($seconds / 60);
}

private function shouldExcludeFailedQueues(): bool
{
$value = $this->configService->get(self::CONFIG_EXCLUDE_FAILED);

// Default true when the setting has never been stored.
return $value === null ? true : (bool) $value;
}

/**
* @return list<string>
*/
private function parseCsvList(string $value): array
{
if (\trim($value) === '') {
return [];
}

$parts = \array_map(\trim(...), \explode(',', $value));

return \array_values(\array_filter($parts, static fn (string $part): bool => $part !== ''));
}

/**
* Parses "async:15, low_priority:60" into queue => grace minutes.
*
* @return array<string, int>
*/
private function parseGraceMap(string $value): array
{
$map = [];
foreach ($this->parseCsvList($value) as $entry) {
if (!\str_contains($entry, ':')) {
continue;
}

[$name, $minutes] = \array_map(\trim(...), \explode(':', $entry, 2));
if ($name === '' || !\is_numeric($minutes)) {
continue;
}

$grace = (int) $minutes;
if ($grace > 0) {
$map[$name] = $grace;
}
} else {
$result = SettingsResult::info('queue', $snippet, 'unknown', $recommended);
}

$collection->add($result);
return $map;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ export default {
},
queue: {
description:
'Messages have been waiting in the message queue longer than the configured grace time. That almost always means no worker is consuming the queue, so asynchronous tasks (mails, indexing, …) pile up.',
'Messages have been waiting in a monitored messenger queue longer than the configured grace time. That almost always means no worker is consuming that queue, so asynchronous tasks (mails, indexing, …) pile up. Failed queues (names containing "failed") are ignored by default because they are usually drained manually.',
solution:
'Make sure a CLI worker is running and supervised (e.g. via systemd or Supervisor) so the queue is drained continuously.',
code: 'bin/console messenger:consume async low_priority failed',
'Make sure a CLI worker is running and supervised (e.g. via systemd or Supervisor) so the queue is drained continuously. Adjust monitored queues and per-queue grace times under Settings → Extensions → Frosh Tools if needed.',
code: 'bin/console messenger:consume async low_priority\n# Extension config examples:\n# monitorQueues = async, low_priority\n# monitorQueueGraceTimes = async:15, low_priority:60\n# monitorExcludeFailedQueues = true',
},
'mysql-timezone': {
description:
Expand Down
35 changes: 31 additions & 4 deletions src/Resources/config/config.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,40 @@

<input-field type="int">
<name>monitorQueueGraceTime</name>
<label>Scheduled queue grace time</label>
<label lang="de-DE">Geplante Karenzzeit der Queue</label>
<helpText>After X minutes a queue is considered stuck / faulty</helpText>
<helpText lang="de-DE">Nach X Minuten gilt eine Queue als festgefahren / fehlerhaft</helpText>
<label>Default queue grace time (minutes)</label>
<label lang="de-DE">Standard-Karenzzeit der Queue (Minuten)</label>
<helpText>After this many minutes a pending message is considered stuck. Used when no per-queue override matches.</helpText>
<helpText lang="de-DE">Nach so vielen Minuten gilt eine ausstehende Nachricht als festgefahren. Gilt, wenn kein Queue-Override greift.</helpText>
<defaultValue>15</defaultValue>
</input-field>

<input-field type="bool">
<name>monitorExcludeFailedQueues</name>
<label>Exclude failed queues</label>
<label lang="de-DE">Failed-Queues ausschließen</label>
<helpText>Ignore messenger queues whose name contains "failed" (e.g. async_failed). Those are usually drained manually and would otherwise cause false positives.</helpText>
<helpText lang="de-DE">Ignoriert Messenger-Queues, deren Name "failed" enthält (z. B. async_failed). Diese werden meist manuell abgearbeitet und würden sonst Fehlalarme auslösen.</helpText>
<defaultValue>true</defaultValue>
</input-field>

<input-field type="text">
<name>monitorQueues</name>
<label>Queues to monitor</label>
<label lang="de-DE">Zu überwachende Queues</label>
<helpText>Optional comma-separated allowlist (e.g. async, low_priority). Leave empty to monitor all queues (except failed ones when exclusion is enabled).</helpText>
<helpText lang="de-DE">Optionale kommaseparierte Allowlist (z. B. async, low_priority). Leer lassen, um alle Queues zu überwachen (außer Failed-Queues, wenn der Ausschluss aktiv ist).</helpText>
<placeholder>async, low_priority</placeholder>
</input-field>

<input-field type="text">
<name>monitorQueueGraceTimes</name>
<label>Per-queue grace times</label>
<label lang="de-DE">Karenzzeiten pro Queue</label>
<helpText>Optional overrides as queue:minutes pairs (e.g. async:15, low_priority:60). Queues not listed use the default grace time.</helpText>
<helpText lang="de-DE">Optionale Overrides als Queue:Minuten-Paare (z. B. async:15, low_priority:60). Nicht gelistete Queues nutzen die Standard-Karenzzeit.</helpText>
<placeholder>async:15, low_priority:60</placeholder>
</input-field>

<input-field type="int">
<name>monitorTaskGraceTime</name>
<label>Scheduled task grace time</label>
Expand Down
Loading
Loading