diff --git a/.gitignore b/.gitignore index f65cb550..109e4dba 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,12 @@ /src/Resources/public/ /src/Resources/app/administration/node_modules/ /src/Resources/app/administration/.tmp/ +# monolog_parser local build only — keep packaged native/lib/*-linux-*.so / *-darwin-*.dylib +/native/monolog_parser/src/*.o +/native/monolog_parser/tests/test_parser +/native/monolog_parser/bench/bench +/native/monolog_parser/libmonolog_parser.so +/native/monolog_parser/libmonolog_parser.dylib +# generic alias from `make install` (optional local convenience) +/native/lib/libmonolog_parser.so +/native/lib/libmonolog_parser.dylib diff --git a/README.md b/README.md index e9c4450c..d8f7a440 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Dedicated list/detail UI for Shopware webhooks (including inline search). | **Queue** | Multi-transport support (Doctrine, Redis, AMQP, fallback); browse without consuming (or fetch+requeue); retry failed messages; purge transport; reset queue. | | **Scheduled Tasks** | List, run, deactivate; edit interval & next execution; register tasks. | | **State Machines** | Diagram viewer for any registered state machine. | -| **Log Viewer** | Read `var/log/*.log` in the browser. | +| **Log Viewer** | Read `var/log/*.log` in the browser. Optional SIMD C parser via PHP FFI (`native/monolog_parser`). | | **Feature Flags** | Inspect and toggle flags from core/plugins. | | **Elasticsearch** | Status, indices, reindex, alias switch, unused/orphaned cleanup, console. Optional `show_all_indices`. | | **Fastly** | Purge all / by URL and basic stats when Fastly is configured. | diff --git a/composer.json b/composer.json index 2df86e88..04aeea66 100644 --- a/composer.json +++ b/composer.json @@ -46,6 +46,9 @@ "require-dev": { "shopware/elasticsearch": "~6.6.0 || ~6.7.0" }, + "suggest": { + "ext-ffi": "Optional: load native/lib/libmonolog_parser.so for SIMD-accelerated Monolog log parsing" + }, "config": { "allow-plugins": { "symfony/runtime": true @@ -54,6 +57,7 @@ "scripts": { "format": "docker run --rm -v $(pwd):/ext shopware/shopware-cli:latest extension format /ext", "check": "docker run --rm -v $(pwd):/ext shopware/shopware-cli:latest extension validate --full /ext", - "phpunit": "../../../vendor/bin/phpunit -c phpunit.xml" + "phpunit": "../../../vendor/bin/phpunit -c phpunit.xml", + "build-native": "make -C native/monolog_parser install" } } diff --git a/native/lib/README.md b/native/lib/README.md new file mode 100644 index 00000000..8f94504d --- /dev/null +++ b/native/lib/README.md @@ -0,0 +1,24 @@ +# Prebuilt `libmonolog_parser` (Zig cross-compile) + +| File | Platform | libc | +| --- | --- | --- | +| `libmonolog_parser-linux-x86_64.so` | Linux amd64 | glibc | +| `libmonolog_parser-linux-arm64.so` | Linux arm64 | glibc | +| `libmonolog_parser-linux-musl-x86_64.so` | Linux amd64 | musl | +| `libmonolog_parser-linux-musl-arm64.so` | Linux arm64 | musl | +| `libmonolog_parser-darwin-x86_64.dylib` | macOS Intel | — | +| `libmonolog_parser-darwin-arm64.dylib` | macOS Apple Silicon | — | + +## Rebuild all (one command) + +```bash +cd ../monolog_parser +make zig-release # needs zig ≥ 0.13 +# or: make zig-release ZIG=/tmp/zig-linux-x86_64-0.13.0/zig +``` + +Produced here with Zig 0.13 from a single Linux host — no Alpine/macOS runners required. + +## Runtime + +PHP `NativeLibraryLocator` auto-selects; users need `ext-ffi` only (optional). diff --git a/native/lib/libmonolog_parser-darwin-arm64.dylib b/native/lib/libmonolog_parser-darwin-arm64.dylib new file mode 100755 index 00000000..f337409b Binary files /dev/null and b/native/lib/libmonolog_parser-darwin-arm64.dylib differ diff --git a/native/lib/libmonolog_parser-darwin-x86_64.dylib b/native/lib/libmonolog_parser-darwin-x86_64.dylib new file mode 100755 index 00000000..b75f0cce Binary files /dev/null and b/native/lib/libmonolog_parser-darwin-x86_64.dylib differ diff --git a/native/lib/libmonolog_parser-linux-arm64.so b/native/lib/libmonolog_parser-linux-arm64.so new file mode 100755 index 00000000..d016ce33 Binary files /dev/null and b/native/lib/libmonolog_parser-linux-arm64.so differ diff --git a/native/lib/libmonolog_parser-linux-musl-arm64.so b/native/lib/libmonolog_parser-linux-musl-arm64.so new file mode 100755 index 00000000..797a0eba Binary files /dev/null and b/native/lib/libmonolog_parser-linux-musl-arm64.so differ diff --git a/native/lib/libmonolog_parser-linux-musl-x86_64.so b/native/lib/libmonolog_parser-linux-musl-x86_64.so new file mode 100755 index 00000000..428dc7e4 Binary files /dev/null and b/native/lib/libmonolog_parser-linux-musl-x86_64.so differ diff --git a/native/lib/libmonolog_parser-linux-x86_64.so b/native/lib/libmonolog_parser-linux-x86_64.so new file mode 100755 index 00000000..05c98b3a Binary files /dev/null and b/native/lib/libmonolog_parser-linux-x86_64.so differ diff --git a/native/lib/monolog_parser.h b/native/lib/monolog_parser.h new file mode 100644 index 00000000..18024ae6 --- /dev/null +++ b/native/lib/monolog_parser.h @@ -0,0 +1,184 @@ +/** + * monolog_parser — SIMD-accelerated Monolog line parser for PHP FFI. + * + * Matches FroshTools LogController LINE_MATCH: + * /\[(?.*)] (?.*)\.(?DEBUG|INFO|…):(?.*)/m + * + * Two usage modes for PHP: + * 1) Parse a line/buffer you already hold (monolog_parse_line / _buffer) + * 2) Open the log file in C and yield/page parsed entries + * monolog_reader_open_backwards → monolog_reader_next (iterator) + * monolog_file_read_backwards (one-shot page) + */ +#ifndef MONOLOG_PARSER_H +#define MONOLOG_PARSER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_WIN32) || defined(__CYGWIN__) +# ifdef MONOLOG_PARSER_BUILD +# define MONOLOG_API __declspec(dllexport) +# else +# define MONOLOG_API __declspec(dllimport) +# endif +#else +# define MONOLOG_API __attribute__((visibility("default"))) +#endif + +/** Parsed field offsets relative to the start of the input string. */ +typedef struct monolog_fields { + int32_t date_off; + int32_t date_len; + int32_t channel_off; + int32_t channel_len; + int32_t level_off; + int32_t level_len; + int32_t message_off; + int32_t message_len; + /** 1 if the Monolog pattern matched, 0 otherwise. */ + int32_t matched; +} monolog_fields_t; + +/** + * One line within a multi-line buffer (offsets relative to buffer start). + * line_len excludes a trailing '\\n' / '\\r\\n'. + */ +typedef struct monolog_line_span { + int32_t off; + int32_t len; +} monolog_line_span_t; + +/** Library version string (static storage). */ +MONOLOG_API const char *monolog_parser_version(void); + +/** + * Which SIMD path is active: "sse2", "neon", or "scalar". + */ +MONOLOG_API const char *monolog_parser_simd(void); + +/** + * Parse a single Monolog line (need not be NUL-terminated). + * Returns 1 on match, 0 on no match. out must be non-NULL. + */ +MONOLOG_API int monolog_parse_line(const char *line, size_t len, monolog_fields_t *out); + +/** + * Find newline byte offsets in [buf, buf+len) using SIMD scan. + * Writes up to max_offsets offsets into offsets[]; returns count found + * (may be > max_offsets if truncated — check return vs max). + * If offsets is NULL, only counts. + */ +MONOLOG_API size_t monolog_find_newlines(const char *buf, size_t len, + size_t *offsets, size_t max_offsets); + +/** + * Split buffer into line spans (no trailing CR/LF). Handles \\n and \\r\\n. + * Returns number of lines written (capped by max_lines). + */ +MONOLOG_API size_t monolog_split_lines(const char *buf, size_t len, + monolog_line_span_t *lines, size_t max_lines); + +/** + * Parse every line in a buffer. entries[i] fields are relative to buf. + * Unmatched lines still get an entry with matched=0; for those the whole + * line is reported as message (message_off/len set, others 0). + * Returns number of lines processed (capped by max_entries). + */ +MONOLOG_API size_t monolog_parse_buffer(const char *buf, size_t len, + monolog_fields_t *entries, size_t max_entries); + +/** + * SIMD memchr — find first occurrence of byte c in [buf, buf+len). + * Returns pointer to the byte, or NULL. + */ +MONOLOG_API const char *monolog_memchr(const char *buf, int c, size_t len); + +/** + * Find last occurrence of byte c in [buf, buf+len). + * Returns pointer to the byte, or NULL. + */ +MONOLOG_API const char *monolog_memrchr(const char *buf, int c, size_t len); + +/* ===================================================================== + * File reader — open in C, yield parsed lines (newest first) + * ===================================================================== */ + +typedef struct monolog_reader monolog_reader_t; + +/** + * Open path for reverse iteration (newest log line first). + * Returns NULL on open failure. + */ +MONOLOG_API monolog_reader_t *monolog_reader_open_backwards(const char *path); + +/** Close reader and free all buffers. Safe on NULL. */ +MONOLOG_API void monolog_reader_close(monolog_reader_t *r); + +/** Total line count (SIMD newline scan on open). */ +MONOLOG_API uint64_t monolog_reader_total_lines(const monolog_reader_t *r); + +/** Last error message, or NULL. */ +MONOLOG_API const char *monolog_reader_error(const monolog_reader_t *r); + +/** + * Yield next line (newest-first), parse it. + * + * Returns: + * 1 — entry written; *line_out points at NUL-terminated line + * (valid until next next()/close). fields offsets relative to *line_out. + * 0 — EOF + * -1 — error (see monolog_reader_error) + * + * Unmatched lines: fields->matched == 0, message = full line. + */ +MONOLOG_API int monolog_reader_next(monolog_reader_t *r, + monolog_fields_t *out, + const char **line_out); + +/** + * Discard the next n lines (newest-first). Returns 1 ok, 0 hit EOF early, -1 error. + */ +MONOLOG_API int monolog_reader_skip(monolog_reader_t *r, uint64_t n); + +/* ===================================================================== + * One-shot page — preferred for LogController offset/limit + * ===================================================================== */ + +/** Owned strings; free via monolog_page_free only. */ +typedef struct monolog_page_entry { + int32_t matched; + const char *date; + const char *channel; + const char *level; + const char *message; +} monolog_page_entry_t; + +typedef struct monolog_page monolog_page_t; + +/** + * Read [offset, offset+limit) lines from end of file (offset 0 = newest), + * parse each. On success *out is allocated; free with monolog_page_free. + * Returns 0 on success, -1 on error. + */ +MONOLOG_API int monolog_file_read_backwards(const char *path, + uint64_t offset, + uint64_t limit, + monolog_page_t **out); + +MONOLOG_API void monolog_page_free(monolog_page_t *page); + +MONOLOG_API uint64_t monolog_page_total_lines(const monolog_page_t *page); +MONOLOG_API uint32_t monolog_page_count(const monolog_page_t *page); +MONOLOG_API const monolog_page_entry_t *monolog_page_entry(const monolog_page_t *page, + uint32_t index); + +#ifdef __cplusplus +} +#endif + +#endif /* MONOLOG_PARSER_H */ diff --git a/native/monolog_parser/Makefile b/native/monolog_parser/Makefile new file mode 100644 index 00000000..351ba64d --- /dev/null +++ b/native/monolog_parser/Makefile @@ -0,0 +1,129 @@ +# monolog_parser — SIMD Monolog library +# +# Native (host toolchain): +# make / make test / make package +# +# Cross-compile ALL release targets with Zig (preferred for CI + local): +# make zig-release # needs `zig` on PATH or ZIG=/path/to/zig +# make zig-release ZIG=... +# +# Artifacts → ../lib/ +# libmonolog_parser-linux-x86_64.so +# libmonolog_parser-linux-arm64.so +# libmonolog_parser-linux-musl-x86_64.so +# libmonolog_parser-linux-musl-arm64.so +# libmonolog_parser-darwin-x86_64.dylib +# libmonolog_parser-darwin-arm64.dylib + +CC ?= cc +CFLAGS ?= -O3 -fPIC -Wall -Wextra -Wpedantic -std=c11 +CFLAGS += -Iinclude +ZIG ?= zig + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ARCH_RAW := $(UNAME_M) +ifeq ($(ARCH_RAW),amd64) + ARCH := x86_64 +else ifeq ($(ARCH_RAW),x86_64) + ARCH := x86_64 +else ifeq ($(ARCH_RAW),aarch64) + ARCH := arm64 +else ifeq ($(ARCH_RAW),arm64) + ARCH := arm64 +else + ARCH := $(ARCH_RAW) +endif + +ifndef LIBC + ifneq ($(wildcard /lib/ld-musl-*.so.1),) + LIBC := musl + else + LIBC := gnu + endif +endif + +ifeq ($(UNAME_S),Darwin) + OS := darwin + LIBEXT := dylib + LIBC := n/a + LDFLAGS += -dynamiclib -install_name @rpath/libmonolog_parser.$(LIBEXT) + CFLAGS += -mmacosx-version-min=11.0 + TEST_RPATH := -Wl,-rpath,@loader_path/.. + LIB_PACKAGED := libmonolog_parser-darwin-$(ARCH).$(LIBEXT) +else + OS := linux + LIBEXT := so + LDFLAGS += -shared + TEST_RPATH := -Wl,-rpath,'$$ORIGIN/..' + ifeq ($(LIBC),musl) + LIB_PACKAGED := libmonolog_parser-linux-musl-$(ARCH).$(LIBEXT) + else + LIB_PACKAGED := libmonolog_parser-linux-$(ARCH).$(LIBEXT) + endif +endif + +ifeq ($(ARCH),arm64) + ifneq ($(UNAME_S),Darwin) + CFLAGS += -march=armv8-a+simd + endif +endif + +SRC := src/simd_scan.c src/monolog_parser.c src/monolog_reader.c +OBJ := $(SRC:.c=.o) + +LIB_LOCAL := libmonolog_parser.$(LIBEXT) +INSTALL_DIR ?= ../lib + +.PHONY: all clean test bench install package shared info zig-release zig-one + +all: shared + +info: + @echo "os=$(OS) arch=$(ARCH) libc=$(LIBC) ext=$(LIBEXT)" + @echo "local=$(LIB_LOCAL) packaged=$(LIB_PACKAGED)" + @echo "zig=$(ZIG)" + +shared: $(LIB_LOCAL) + +$(LIB_LOCAL): $(OBJ) + $(CC) $(LDFLAGS) -o $@ $^ + @echo "built $@" + +src/%.o: src/%.c include/monolog_parser.h + $(CC) $(CFLAGS) -DMONOLOG_PARSER_BUILD -c -o $@ $< + +tests/test_parser: tests/test_parser.c $(LIB_LOCAL) + $(CC) $(CFLAGS) -o $@ tests/test_parser.c -L. -lmonolog_parser $(TEST_RPATH) + +bench/bench: bench/bench.c $(LIB_LOCAL) + $(CC) $(CFLAGS) -o $@ bench/bench.c -L. -lmonolog_parser $(TEST_RPATH) + +test: tests/test_parser + LD_LIBRARY_PATH="$(CURDIR):$$LD_LIBRARY_PATH" ./tests/test_parser + +bench: bench/bench + LD_LIBRARY_PATH="$(CURDIR):$$LD_LIBRARY_PATH" ./bench/bench 100000 + +package: $(LIB_LOCAL) + mkdir -p $(INSTALL_DIR) + cp -f $(LIB_LOCAL) $(INSTALL_DIR)/$(LIB_PACKAGED) + cp -f include/monolog_parser.h $(INSTALL_DIR)/ + @echo "packaged $(INSTALL_DIR)/$(LIB_PACKAGED) [libc=$(LIBC)]" + +install: package + cp -f $(LIB_LOCAL) $(INSTALL_DIR)/$(LIB_LOCAL) + +# ---- Zig cross-compile (all platforms from one machine) ------------------ + +zig-release: + ZIG="$(ZIG)" OUT="$(abspath $(INSTALL_DIR))" sh ./build-zig.sh + +# make zig-one TARGET=linux-musl-arm64 +zig-one: + @test -n "$(TARGET)" || { echo "usage: make zig-one TARGET=linux-musl-arm64"; exit 1; } + ZIG="$(ZIG)" OUT="$(abspath $(INSTALL_DIR))" sh ./build-zig.sh $(TARGET) + +clean: + rm -f $(OBJ) $(LIB_LOCAL) tests/test_parser bench/bench diff --git a/native/monolog_parser/README.md b/native/monolog_parser/README.md new file mode 100644 index 00000000..b5115c55 --- /dev/null +++ b/native/monolog_parser/README.md @@ -0,0 +1,49 @@ +# monolog_parser + +SIMD Monolog file reader for FroshTools — **prebuilt** shared libs for PHP FFI. +**End users never compile.** + +## Cross-compile everything with Zig (recommended) + +One machine, six artifacts: + +```bash +# zig on PATH, or: make zig-release ZIG=/path/to/zig +make zig-release + +# single target: +make zig-one TARGET=linux-musl-arm64 +``` + +`build-zig.sh` maps: + +| Artifact | Zig target | +| --- | --- | +| `libmonolog_parser-linux-x86_64.so` | `x86_64-linux-gnu` | +| `libmonolog_parser-linux-arm64.so` | `aarch64-linux-gnu` | +| `libmonolog_parser-linux-musl-x86_64.so` | `x86_64-linux-musl` | +| `libmonolog_parser-linux-musl-arm64.so` | `aarch64-linux-musl` | +| `libmonolog_parser-darwin-x86_64.dylib` | `x86_64-macos-none` | +| `libmonolog_parser-darwin-arm64.dylib` | `aarch64-macos-none` | + +CI (`.github/workflows/native-monolog-parser.yml`) installs Zig 0.13 and runs `make zig-release` on a single `ubuntu-22.04` runner. + +## Runtime selection + +`NativeLibraryLocator` picks OS + arch + musl/glibc automatically. +Override: `MONOLOG_PARSER_LIB`, `MONOLOG_PARSER_LIBC=musl|gnu`. + +## Host-only build (optional) + +```bash +make test package # current OS/arch only +``` + +## API + +```c +monolog_file_read_backwards(path, offset, limit, &page); +// iterator: monolog_reader_open_backwards → next → close +``` + +See `include/monolog_parser.h`. diff --git a/native/monolog_parser/bench/bench.c b/native/monolog_parser/bench/bench.c new file mode 100644 index 00000000..1ad8982c --- /dev/null +++ b/native/monolog_parser/bench/bench.c @@ -0,0 +1,104 @@ +/** + * Micro-benchmark: SIMD monolog_parse_buffer throughput. + */ +#define _POSIX_C_SOURCE 199309L + +#include "monolog_parser.h" + +#include +#include +#include +#include + +static double now_sec(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +} + +static char *build_corpus(size_t lines, size_t *out_len) +{ + static const char *tpl = + "[2024-03-15T10:30:00.%06zu+00:00] request.ERROR: Uncaught PHP Exception " + "RuntimeException: \"Something broke at line %zu\" at /var/www/html/src/Foo.php " + "line 42 {\"exception\":[]} []\n"; + size_t cap = lines * 256; + char *buf = malloc(cap); + if (!buf) { + return NULL; + } + size_t used = 0; + for (size_t i = 0; i < lines; i++) { + char line[320]; + int n = snprintf(line, sizeof(line), tpl, i % 1000000, i); + if (n < 0) { + free(buf); + return NULL; + } + if (used + (size_t)n > cap) { + cap *= 2; + char *nb = realloc(buf, cap); + if (!nb) { + free(buf); + return NULL; + } + buf = nb; + } + memcpy(buf + used, line, (size_t)n); + used += (size_t)n; + } + *out_len = used; + return buf; +} + +int main(int argc, char **argv) +{ + size_t lines = 100000; + if (argc > 1) { + lines = (size_t)strtoull(argv[1], NULL, 10); + } + + size_t len = 0; + char *buf = build_corpus(lines, &len); + if (!buf) { + fprintf(stderr, "alloc failed\n"); + return 1; + } + + monolog_fields_t *entries = calloc(lines, sizeof(*entries)); + if (!entries) { + free(buf); + return 1; + } + + printf("corpus: %zu lines, %zu bytes, simd=%s, version=%s\n", + lines, len, monolog_parser_simd(), monolog_parser_version()); + + /* warmup */ + (void)monolog_parse_buffer(buf, len, entries, lines); + + const int rounds = 20; + double t0 = now_sec(); + size_t parsed = 0; + for (int r = 0; r < rounds; r++) { + parsed = monolog_parse_buffer(buf, len, entries, lines); + } + double t1 = now_sec(); + double elapsed = t1 - t0; + double lines_per_sec = ((double)lines * (double)rounds) / elapsed; + double mb_per_sec = ((double)len * (double)rounds / (1024.0 * 1024.0)) / elapsed; + + size_t matched = 0; + for (size_t i = 0; i < parsed; i++) { + matched += (size_t)entries[i].matched; + } + + printf("parsed=%zu matched=%zu rounds=%d time=%.4fs\n", parsed, matched, rounds, elapsed); + printf("throughput: %.2f M lines/s | %.2f MiB/s\n", + lines_per_sec / 1e6, mb_per_sec); + + free(entries); + free(buf); + return 0; +} diff --git a/native/monolog_parser/build-zig.sh b/native/monolog_parser/build-zig.sh new file mode 100644 index 00000000..414c6210 --- /dev/null +++ b/native/monolog_parser/build-zig.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env sh +# Cross-compile libmonolog_parser for all release platforms with Zig. +# +# Usage: +# ./build-zig.sh # all targets → ../lib/ +# ./build-zig.sh linux-musl-arm64 +# ZIG=/path/to/zig ./build-zig.sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname "$0")" && pwd) +OUT="${OUT:-$ROOT/../lib}" +SRC="$ROOT/src/simd_scan.c $ROOT/src/monolog_parser.c $ROOT/src/monolog_reader.c" +INC="-I$ROOT/include" +# Keep flags portable across zig targets (no host -march) +CFLAGS_COMMON="-O3 -fPIC -std=c11 -Wall -Wextra -DMONOLOG_PARSER_BUILD" + +ZIG="${ZIG:-zig}" +if ! command -v "$ZIG" >/dev/null 2>&1; then + for c in \ + "$ROOT/../../.tools/zig/zig" \ + /tmp/zig-linux-x86_64-*/zig \ + /opt/zig/zig + do + for hit in $c; do + if [ -x "$hit" ]; then + ZIG=$hit + break 2 + fi + done + done +fi + +if ! command -v "$ZIG" >/dev/null 2>&1 && [ ! -x "$ZIG" ]; then + echo "zig not found — install from https://ziglang.org or set ZIG=" >&2 + exit 1 +fi + +echo "using: $ZIG ($("$ZIG" version 2>/dev/null || true))" +mkdir -p "$OUT" + +# name|zig-target|ext +# aarch64 NEON is baseline — no extra -march needed (zig rejects gcc-style -march) +TARGETS=" +linux-x86_64|x86_64-linux-gnu|so +linux-arm64|aarch64-linux-gnu|so +linux-musl-x86_64|x86_64-linux-musl|so +linux-musl-arm64|aarch64-linux-musl|so +darwin-x86_64|x86_64-macos-none|dylib +darwin-arm64|aarch64-macos-none|dylib +" + +build_one() { + name=$1 + target=$2 + ext=$3 + out="$OUT/libmonolog_parser-${name}.${ext}" + + echo "→ $name ($target) → $out" + + # shellcheck disable=SC2086 + "$ZIG" cc \ + -target "$target" \ + -shared \ + $CFLAGS_COMMON \ + $INC \ + -o "$out" \ + $SRC + + ls -la "$out" +} + +FILTER="${1:-}" + +echo "$TARGETS" | while IFS= read -r line; do + [ -z "$line" ] && continue + name=$(echo "$line" | cut -d'|' -f1) + target=$(echo "$line" | cut -d'|' -f2) + ext=$(echo "$line" | cut -d'|' -f3) + + if [ -n "$FILTER" ] && [ "$FILTER" != "$name" ]; then + continue + fi + + build_one "$name" "$target" "$ext" +done + +cp -f "$ROOT/include/monolog_parser.h" "$OUT/" + +echo "" +echo "Done. Artifacts in $OUT:" +ls -la "$OUT"/libmonolog_parser-* 2>/dev/null || true diff --git a/native/monolog_parser/include/monolog_parser.h b/native/monolog_parser/include/monolog_parser.h new file mode 100644 index 00000000..18024ae6 --- /dev/null +++ b/native/monolog_parser/include/monolog_parser.h @@ -0,0 +1,184 @@ +/** + * monolog_parser — SIMD-accelerated Monolog line parser for PHP FFI. + * + * Matches FroshTools LogController LINE_MATCH: + * /\[(?.*)] (?.*)\.(?DEBUG|INFO|…):(?.*)/m + * + * Two usage modes for PHP: + * 1) Parse a line/buffer you already hold (monolog_parse_line / _buffer) + * 2) Open the log file in C and yield/page parsed entries + * monolog_reader_open_backwards → monolog_reader_next (iterator) + * monolog_file_read_backwards (one-shot page) + */ +#ifndef MONOLOG_PARSER_H +#define MONOLOG_PARSER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_WIN32) || defined(__CYGWIN__) +# ifdef MONOLOG_PARSER_BUILD +# define MONOLOG_API __declspec(dllexport) +# else +# define MONOLOG_API __declspec(dllimport) +# endif +#else +# define MONOLOG_API __attribute__((visibility("default"))) +#endif + +/** Parsed field offsets relative to the start of the input string. */ +typedef struct monolog_fields { + int32_t date_off; + int32_t date_len; + int32_t channel_off; + int32_t channel_len; + int32_t level_off; + int32_t level_len; + int32_t message_off; + int32_t message_len; + /** 1 if the Monolog pattern matched, 0 otherwise. */ + int32_t matched; +} monolog_fields_t; + +/** + * One line within a multi-line buffer (offsets relative to buffer start). + * line_len excludes a trailing '\\n' / '\\r\\n'. + */ +typedef struct monolog_line_span { + int32_t off; + int32_t len; +} monolog_line_span_t; + +/** Library version string (static storage). */ +MONOLOG_API const char *monolog_parser_version(void); + +/** + * Which SIMD path is active: "sse2", "neon", or "scalar". + */ +MONOLOG_API const char *monolog_parser_simd(void); + +/** + * Parse a single Monolog line (need not be NUL-terminated). + * Returns 1 on match, 0 on no match. out must be non-NULL. + */ +MONOLOG_API int monolog_parse_line(const char *line, size_t len, monolog_fields_t *out); + +/** + * Find newline byte offsets in [buf, buf+len) using SIMD scan. + * Writes up to max_offsets offsets into offsets[]; returns count found + * (may be > max_offsets if truncated — check return vs max). + * If offsets is NULL, only counts. + */ +MONOLOG_API size_t monolog_find_newlines(const char *buf, size_t len, + size_t *offsets, size_t max_offsets); + +/** + * Split buffer into line spans (no trailing CR/LF). Handles \\n and \\r\\n. + * Returns number of lines written (capped by max_lines). + */ +MONOLOG_API size_t monolog_split_lines(const char *buf, size_t len, + monolog_line_span_t *lines, size_t max_lines); + +/** + * Parse every line in a buffer. entries[i] fields are relative to buf. + * Unmatched lines still get an entry with matched=0; for those the whole + * line is reported as message (message_off/len set, others 0). + * Returns number of lines processed (capped by max_entries). + */ +MONOLOG_API size_t monolog_parse_buffer(const char *buf, size_t len, + monolog_fields_t *entries, size_t max_entries); + +/** + * SIMD memchr — find first occurrence of byte c in [buf, buf+len). + * Returns pointer to the byte, or NULL. + */ +MONOLOG_API const char *monolog_memchr(const char *buf, int c, size_t len); + +/** + * Find last occurrence of byte c in [buf, buf+len). + * Returns pointer to the byte, or NULL. + */ +MONOLOG_API const char *monolog_memrchr(const char *buf, int c, size_t len); + +/* ===================================================================== + * File reader — open in C, yield parsed lines (newest first) + * ===================================================================== */ + +typedef struct monolog_reader monolog_reader_t; + +/** + * Open path for reverse iteration (newest log line first). + * Returns NULL on open failure. + */ +MONOLOG_API monolog_reader_t *monolog_reader_open_backwards(const char *path); + +/** Close reader and free all buffers. Safe on NULL. */ +MONOLOG_API void monolog_reader_close(monolog_reader_t *r); + +/** Total line count (SIMD newline scan on open). */ +MONOLOG_API uint64_t monolog_reader_total_lines(const monolog_reader_t *r); + +/** Last error message, or NULL. */ +MONOLOG_API const char *monolog_reader_error(const monolog_reader_t *r); + +/** + * Yield next line (newest-first), parse it. + * + * Returns: + * 1 — entry written; *line_out points at NUL-terminated line + * (valid until next next()/close). fields offsets relative to *line_out. + * 0 — EOF + * -1 — error (see monolog_reader_error) + * + * Unmatched lines: fields->matched == 0, message = full line. + */ +MONOLOG_API int monolog_reader_next(monolog_reader_t *r, + monolog_fields_t *out, + const char **line_out); + +/** + * Discard the next n lines (newest-first). Returns 1 ok, 0 hit EOF early, -1 error. + */ +MONOLOG_API int monolog_reader_skip(monolog_reader_t *r, uint64_t n); + +/* ===================================================================== + * One-shot page — preferred for LogController offset/limit + * ===================================================================== */ + +/** Owned strings; free via monolog_page_free only. */ +typedef struct monolog_page_entry { + int32_t matched; + const char *date; + const char *channel; + const char *level; + const char *message; +} monolog_page_entry_t; + +typedef struct monolog_page monolog_page_t; + +/** + * Read [offset, offset+limit) lines from end of file (offset 0 = newest), + * parse each. On success *out is allocated; free with monolog_page_free. + * Returns 0 on success, -1 on error. + */ +MONOLOG_API int monolog_file_read_backwards(const char *path, + uint64_t offset, + uint64_t limit, + monolog_page_t **out); + +MONOLOG_API void monolog_page_free(monolog_page_t *page); + +MONOLOG_API uint64_t monolog_page_total_lines(const monolog_page_t *page); +MONOLOG_API uint32_t monolog_page_count(const monolog_page_t *page); +MONOLOG_API const monolog_page_entry_t *monolog_page_entry(const monolog_page_t *page, + uint32_t index); + +#ifdef __cplusplus +} +#endif + +#endif /* MONOLOG_PARSER_H */ diff --git a/native/monolog_parser/src/monolog_parser.c b/native/monolog_parser/src/monolog_parser.c new file mode 100644 index 00000000..25f125a2 --- /dev/null +++ b/native/monolog_parser/src/monolog_parser.c @@ -0,0 +1,257 @@ +/** + * Monolog line parser — mirrors FroshTools PCRE LINE_MATCH semantics. + * + * Pattern: + * \[ (?.*) ][ ] (?.*) \. (?LEVEL) : (?.*) + * + * channel is greedy → rightmost ".LEVEL:" wins (same as PCRE). + */ +#include "monolog_parser.h" + +#include + +#define MONOLOG_PARSER_VERSION_STR "1.0.0" + +typedef struct { + const char *name; + int len; +} level_def_t; + +/* Longest first (levels are prefix-free). */ +static const level_def_t LEVELS[] = { + {"EMERGENCY", 9}, + {"CRITICAL", 8}, + {"WARNING", 7}, + {"NOTICE", 6}, + {"DEBUG", 5}, + {"ERROR", 5}, + {"ALERT", 5}, + {"INFO", 4}, +}; +static const size_t N_LEVELS = sizeof(LEVELS) / sizeof(LEVELS[0]); + +const char *monolog_parser_version(void) +{ + return MONOLOG_PARSER_VERSION_STR; +} + +static int level_at(const char *s, size_t avail, int *out_len) +{ + for (size_t i = 0; i < N_LEVELS; i++) { + int n = LEVELS[i].len; + if ((size_t)n + 1 > avail) { + continue; + } + if (memcmp(s, LEVELS[i].name, (size_t)n) == 0 && s[n] == ':') { + *out_len = n; + return 1; + } + } + return 0; +} + +/** + * Find rightmost ".LEVEL:" in [start, start+len). + */ +static int find_rightmost_level(const char *start, size_t len, + const char **level_name, int *level_len, + const char **dot_pos) +{ + const char *best_dot = NULL; + const char *best_level = NULL; + int best_llen = 0; + + const char *p = start; + const char *end = start + len; + + while (p < end) { + const char *dot = monolog_memchr(p, '.', (size_t)(end - p)); + if (!dot) { + break; + } + size_t avail = (size_t)(end - (dot + 1)); + int llen = 0; + if (avail > 0 && level_at(dot + 1, avail, &llen)) { + best_dot = dot; + best_level = dot + 1; + best_llen = llen; + } + p = dot + 1; + } + + if (!best_dot) { + return 0; + } + *dot_pos = best_dot; + *level_name = best_level; + *level_len = best_llen; + return 1; +} + +int monolog_parse_line(const char *line, size_t len, monolog_fields_t *out) +{ + if (!out) { + return 0; + } + memset(out, 0, sizeof(*out)); + + if (!line || len == 0) { + return 0; + } + + /* Strip trailing CR (from CRLF). */ + if (line[len - 1] == '\r') { + len--; + } + if (len == 0) { + return 0; + } + + /* 1) '[' date start */ + const char *lb = monolog_memchr(line, '[', len); + if (!lb) { + return 0; + } + + /* 2) first "] " after '[' */ + const char *rb = NULL; + { + const char *scan = lb + 1; + while (scan < line + len) { + const char *cand = monolog_memchr(scan, ']', (size_t)(line + len - scan)); + if (!cand) { + break; + } + if ((size_t)(line + len - cand) >= 2 && cand[1] == ' ') { + rb = cand; + break; + } + scan = cand + 1; + } + } + if (!rb) { + return 0; + } + + const char *date_start = lb + 1; + size_t date_len = (size_t)(rb - date_start); + const char *body = rb + 2; /* skip "] " */ + size_t body_len = (size_t)(line + len - body); + + /* 3) rightmost ".LEVEL:" in body (PCRE greedy channel) */ + const char *level_name = NULL; + int level_len = 0; + const char *dot = NULL; + if (!find_rightmost_level(body, body_len, &level_name, &level_len, &dot)) { + return 0; + } + + const char *channel_start = body; + size_t channel_len = (size_t)(dot - body); + const char *message_start = level_name + level_len + 1; /* skip "LEVEL:" */ + size_t message_len = (size_t)(line + len - message_start); + + out->date_off = (int32_t)(date_start - line); + out->date_len = (int32_t)date_len; + out->channel_off = (int32_t)(channel_start - line); + out->channel_len = (int32_t)channel_len; + out->level_off = (int32_t)(level_name - line); + out->level_len = (int32_t)level_len; + out->message_off = (int32_t)(message_start - line); + out->message_len = (int32_t)message_len; + out->matched = 1; + return 1; +} + +size_t monolog_split_lines(const char *buf, size_t len, + monolog_line_span_t *lines, size_t max_lines) +{ + if (!buf || !lines || max_lines == 0) { + return 0; + } + + size_t n = 0; + size_t start = 0; + + while (n < max_lines) { + const char *hit = NULL; + if (start < len) { + hit = monolog_memchr(buf + start, '\n', len - start); + } + size_t end = hit ? (size_t)(hit - buf) : len; + size_t line_len = end - start; + if (line_len > 0 && buf[start + line_len - 1] == '\r') { + line_len--; + } + /* skip pure empty trailing line caused by final newline */ + if (!hit && line_len == 0 && n > 0) { + break; + } + lines[n].off = (int32_t)start; + lines[n].len = (int32_t)line_len; + n++; + if (!hit) { + break; + } + start = end + 1; + if (start >= len) { + break; + } + } + return n; +} + +size_t monolog_parse_buffer(const char *buf, size_t len, + monolog_fields_t *entries, size_t max_entries) +{ + if (!buf || !entries || max_entries == 0) { + return 0; + } + + size_t produced = 0; + size_t start = 0; + + while (produced < max_entries) { + const char *hit = NULL; + if (start < len) { + hit = monolog_memchr(buf + start, '\n', len - start); + } + size_t end = hit ? (size_t)(hit - buf) : len; + size_t line_len = end - start; + if (line_len > 0 && buf[start + line_len - 1] == '\r') { + line_len--; + } + + if (!hit && line_len == 0 && produced > 0) { + break; + } + if (start >= len && line_len == 0) { + break; + } + + monolog_fields_t local; + int ok = monolog_parse_line(buf + start, line_len, &local); + if (ok) { + local.date_off += (int32_t)start; + local.channel_off += (int32_t)start; + local.level_off += (int32_t)start; + local.message_off += (int32_t)start; + entries[produced] = local; + } else { + memset(&entries[produced], 0, sizeof(entries[produced])); + entries[produced].matched = 0; + entries[produced].message_off = (int32_t)start; + entries[produced].message_len = (int32_t)line_len; + } + produced++; + + if (!hit) { + break; + } + start = end + 1; + if (start >= len) { + break; + } + } + return produced; +} diff --git a/native/monolog_parser/src/monolog_reader.c b/native/monolog_parser/src/monolog_reader.c new file mode 100644 index 00000000..fff1d5c5 --- /dev/null +++ b/native/monolog_parser/src/monolog_reader.c @@ -0,0 +1,584 @@ +/** + * Reverse log file reader — open file in C, yield/page parsed lines for PHP FFI. + * + * Chunked reverse scan mirrors Frosh\Tools\Components\LineReader::readLinesBackwards, + * then monolog_parse_line on each yielded line. + */ +#include "monolog_parser.h" + +#include +#include +#include + +#define CHUNK_SIZE 65536 + +struct monolog_reader { + FILE *fp; + size_t file_size; + size_t pos; /* byte offset of next older unread region start */ + + char *chunk; + size_t chunk_cap; + + /* buffer[0] from LineReader — incomplete oldest fragment, or sole remaining line */ + char *head; + size_t head_len; + size_t head_cap; + int has_head; + + /* complete lines ready to yield, oldest→newest; pop from end */ + char **ready; + size_t ready_n; + size_t ready_cap; + + char *current; + size_t current_cap; + + uint64_t total_lines; + int first_chunk; + int eof; + int err; + char errbuf[128]; +}; + +struct monolog_page { + uint64_t total_lines; + uint32_t count; + monolog_page_entry_t *entries; +}; + +static int set_err(monolog_reader_t *r, const char *msg) +{ + r->err = 1; + snprintf(r->errbuf, sizeof(r->errbuf), "%s", msg); + return -1; +} + +static int ready_push(monolog_reader_t *r, char *line) +{ + if (r->ready_n + 1 > r->ready_cap) { + size_t ncap = r->ready_cap ? r->ready_cap * 2 : 16; + char **nr = (char **)realloc(r->ready, ncap * sizeof(char *)); + if (!nr) { + free(line); + return set_err(r, "oom ready"); + } + r->ready = nr; + r->ready_cap = ncap; + } + r->ready[r->ready_n++] = line; + return 0; +} + +static int set_head(monolog_reader_t *r, const char *s, size_t len) +{ + if (r->head_cap < len + 1) { + free(r->head); + r->head = (char *)malloc(len + 1); + if (!r->head) { + return set_err(r, "oom head"); + } + r->head_cap = len + 1; + } + memcpy(r->head, s, len); + r->head[len] = '\0'; + r->head_len = len; + r->has_head = 1; + return 0; +} + +static char *dup_slice(const char *s, size_t len) +{ + char *p = (char *)malloc(len + 1); + if (!p) { + return NULL; + } + memcpy(p, s, len); + p[len] = '\0'; + return p; +} + +static uint64_t count_lines_fp(FILE *fp, size_t file_size) +{ + if (file_size == 0) { + return 0; + } + + char buf[CHUNK_SIZE]; + uint64_t newlines = 0; + size_t got_total = 0; + int last_was_nl = 0; + + if (fseek(fp, 0, SEEK_SET) != 0) { + return 0; + } + + while (got_total < file_size) { + size_t want = file_size - got_total; + if (want > sizeof(buf)) { + want = sizeof(buf); + } + size_t n = fread(buf, 1, want, fp); + if (n == 0) { + break; + } + newlines += (uint64_t)monolog_find_newlines(buf, n, NULL, 0); + last_was_nl = (buf[n - 1] == '\n'); + got_total += n; + } + + if (!last_was_nl) { + newlines += 1; + } + return newlines; +} + +/** + * Explode `data` by '\\n' into ready[] + head, matching LineReader: + * - first chunk: strip one trailing '\\n' + * - if has_head: data is older_chunk + head + * - all parts go into a temporary list; last part becomes head if pos>0 + * wait — LineReader keeps buffer as full explode; yields when count>1 from end, + * only buffer[0] is held when needing more. So after explode L0..Lk: + * ready gets L1..Lk (to be popped from end = Lk first), head = L0 if more data + * or if pos==0 everything including L0 is ready. + */ +static int explode_into_reader(monolog_reader_t *r, char *data, size_t len, int at_file_start) +{ + /* strip one trailing newline on first chunk only (LineReader) */ + if (r->first_chunk) { + r->first_chunk = 0; + if (len > 0 && data[len - 1] == '\n') { + len--; + if (len > 0 && data[len - 1] == '\r') { + len--; + } + } + } + + size_t start = 0; + size_t part_count = 0; + /* first pass count */ + for (size_t i = 0; i < len; i++) { + if (data[i] == '\n') { + part_count++; + } + } + part_count += 1; /* last segment */ + + char **parts = (char **)calloc(part_count, sizeof(char *)); + if (!parts) { + return set_err(r, "oom parts"); + } + + size_t pi = 0; + for (size_t i = 0; i <= len; i++) { + if (i == len || data[i] == '\n') { + size_t plen = i - start; + if (plen > 0 && data[start + plen - 1] == '\r') { + plen--; + } + parts[pi] = dup_slice(data + start, plen); + if (!parts[pi]) { + for (size_t j = 0; j < pi; j++) { + free(parts[j]); + } + free(parts); + return set_err(r, "oom part"); + } + pi++; + start = i + 1; + } + } + + /* + * parts[0] .. parts[pi-1] are oldest → newest within this combined chunk. + * If not at file start, parts[0] may be incomplete → becomes head. + * Remaining parts are complete lines → push to ready (still oldest→newest). + */ + size_t from = 0; + if (!at_file_start) { + /* keep parts[0] as head for next older merge */ + if (set_head(r, parts[0], strlen(parts[0])) != 0) { + for (size_t j = 0; j < pi; j++) { + free(parts[j]); + } + free(parts); + return -1; + } + free(parts[0]); + from = 1; + } else { + r->has_head = 0; + } + + for (size_t j = from; j < pi; j++) { + if (ready_push(r, parts[j]) != 0) { + for (size_t k = j + 1; k < pi; k++) { + free(parts[k]); + } + free(parts); + return -1; + } + /* ownership transferred */ + } + free(parts); + return 0; +} + +static int load_older_chunk(monolog_reader_t *r) +{ + if (r->pos == 0) { + /* flush head as final line */ + if (r->has_head) { + char *line = dup_slice(r->head, r->head_len); + if (!line) { + return set_err(r, "oom head flush"); + } + r->has_head = 0; + if (ready_push(r, line) != 0) { + return -1; + } + return 0; + } + r->eof = 1; + return 0; + } + + size_t nread = r->pos > CHUNK_SIZE ? CHUNK_SIZE : r->pos; + size_t start = r->pos - nread; + + if (r->chunk_cap < nread + (r->has_head ? r->head_len : 0) + 1) { + free(r->chunk); + r->chunk_cap = nread + (r->has_head ? r->head_len : 0) + 1; + r->chunk = (char *)malloc(r->chunk_cap); + if (!r->chunk) { + return set_err(r, "oom chunk"); + } + } + + if (fseek(r->fp, (long)start, SEEK_SET) != 0) { + return set_err(r, "fseek failed"); + } + size_t got = fread(r->chunk, 1, nread, r->fp); + if (got != nread) { + return set_err(r, "fread failed"); + } + + /* combine older_chunk + head (LineReader: chunk . buffer[0]) */ + size_t combined_len = got; + if (r->has_head) { + memcpy(r->chunk + got, r->head, r->head_len); + combined_len = got + r->head_len; + r->has_head = 0; + } + r->chunk[combined_len] = '\0'; + + r->pos = start; + int at_start = (r->pos == 0); + + return explode_into_reader(r, r->chunk, combined_len, at_start); +} + +static int ensure_ready(monolog_reader_t *r) +{ + while (r->ready_n == 0 && !r->eof && !r->err) { + /* + * LineReader yields when count(buffer) > 1, else loads more. + * We only keep complete lines in ready[]; head is incomplete. + * So ready_n==0 ⇒ need older chunk (or flush head at pos==0). + */ + if (load_older_chunk(r) != 0) { + return -1; + } + } + return r->err ? -1 : 0; +} + +/* ---- public ----------------------------------------------------------- */ + +monolog_reader_t *monolog_reader_open_backwards(const char *path) +{ + if (!path) { + return NULL; + } + + monolog_reader_t *r = (monolog_reader_t *)calloc(1, sizeof(*r)); + if (!r) { + return NULL; + } + + r->fp = fopen(path, "rb"); + if (!r->fp) { + free(r); + return NULL; + } + + if (fseek(r->fp, 0, SEEK_END) != 0) { + fclose(r->fp); + free(r); + return NULL; + } + long sz = ftell(r->fp); + if (sz < 0) { + fclose(r->fp); + free(r); + return NULL; + } + + r->file_size = (size_t)sz; + r->pos = r->file_size; + r->first_chunk = 1; + r->total_lines = count_lines_fp(r->fp, r->file_size); + + if (r->file_size == 0) { + r->eof = 1; + } + + return r; +} + +void monolog_reader_close(monolog_reader_t *r) +{ + if (!r) { + return; + } + if (r->fp) { + fclose(r->fp); + } + free(r->chunk); + free(r->head); + free(r->current); + for (size_t i = 0; i < r->ready_n; i++) { + free(r->ready[i]); + } + free(r->ready); + free(r); +} + +uint64_t monolog_reader_total_lines(const monolog_reader_t *r) +{ + return r ? r->total_lines : 0; +} + +const char *monolog_reader_error(const monolog_reader_t *r) +{ + if (!r || !r->err) { + return NULL; + } + return r->errbuf; +} + +int monolog_reader_next(monolog_reader_t *r, monolog_fields_t *out, const char **line_out) +{ + if (!r || !out || !line_out) { + return -1; + } + if (r->err) { + return -1; + } + + /* Prefer ready lines; if only head and more file, LineReader would load more + before yielding head — ensure_ready handles that. If ready empty and head + is sole line with pos==0, load_older flushes head into ready. */ + if (ensure_ready(r) != 0) { + return -1; + } + + /* LineReader: if count > 1 pop; if count == 1 and pos==0 yield last. + We never put incomplete head into ready, so pop any ready line. */ + if (r->ready_n == 0) { + return 0; + } + + char *line = r->ready[--r->ready_n]; + size_t len = strlen(line); + + if (r->current_cap < len + 1) { + free(r->current); + r->current = (char *)malloc(len + 1); + if (!r->current) { + free(line); + set_err(r, "oom current"); + return -1; + } + r->current_cap = len + 1; + } + memcpy(r->current, line, len + 1); + free(line); + + memset(out, 0, sizeof(*out)); + if (!monolog_parse_line(r->current, len, out)) { + out->matched = 0; + out->message_off = 0; + out->message_len = (int32_t)len; + } + + *line_out = r->current; + return 1; +} + +int monolog_reader_skip(monolog_reader_t *r, uint64_t n) +{ + if (!r) { + return -1; + } + monolog_fields_t tmp; + const char *line = NULL; + for (uint64_t i = 0; i < n; i++) { + int rc = monolog_reader_next(r, &tmp, &line); + if (rc <= 0) { + return rc; + } + } + return 1; +} + +static char *xstrndup(const char *s, size_t n) +{ + char *p = (char *)malloc(n + 1); + if (!p) { + return NULL; + } + memcpy(p, s, n); + p[n] = '\0'; + return p; +} + +int monolog_file_read_backwards(const char *path, uint64_t offset, uint64_t limit, + monolog_page_t **out) +{ + if (!path || !out) { + return -1; + } + *out = NULL; + + monolog_reader_t *r = monolog_reader_open_backwards(path); + if (!r) { + return -1; + } + + monolog_page_t *page = (monolog_page_t *)calloc(1, sizeof(*page)); + if (!page) { + monolog_reader_close(r); + return -1; + } + page->total_lines = monolog_reader_total_lines(r); + + if (limit == 0) { + monolog_reader_close(r); + *out = page; + return 0; + } + + if (offset > 0) { + int src = monolog_reader_skip(r, offset); + if (src < 0) { + monolog_page_free(page); + monolog_reader_close(r); + return -1; + } + } + + monolog_page_entry_t *entries = + (monolog_page_entry_t *)calloc((size_t)limit, sizeof(*entries)); + if (!entries) { + monolog_page_free(page); + monolog_reader_close(r); + return -1; + } + + uint32_t n = 0; + for (uint64_t i = 0; i < limit; i++) { + monolog_fields_t f; + const char *line = NULL; + int rc = monolog_reader_next(r, &f, &line); + if (rc < 0) { + for (uint32_t j = 0; j < n; j++) { + free((void *)entries[j].date); + free((void *)entries[j].channel); + free((void *)entries[j].level); + free((void *)entries[j].message); + } + free(entries); + monolog_page_free(page); + monolog_reader_close(r); + return -1; + } + if (rc == 0) { + break; + } + + monolog_page_entry_t *e = &entries[n]; + e->matched = f.matched; + if (f.matched) { + e->date = xstrndup(line + f.date_off, (size_t)f.date_len); + e->channel = xstrndup(line + f.channel_off, (size_t)f.channel_len); + e->level = xstrndup(line + f.level_off, (size_t)f.level_len); + e->message = xstrndup(line + f.message_off, (size_t)f.message_len); + } else { + e->date = xstrndup("unknown", 7); + e->channel = xstrndup("unknown", 7); + e->level = xstrndup("unknown", 7); + e->message = xstrndup(line, strlen(line)); + } + if (!e->date || !e->channel || !e->level || !e->message) { + free((void *)e->date); + free((void *)e->channel); + free((void *)e->level); + free((void *)e->message); + for (uint32_t j = 0; j < n; j++) { + free((void *)entries[j].date); + free((void *)entries[j].channel); + free((void *)entries[j].level); + free((void *)entries[j].message); + } + free(entries); + monolog_page_free(page); + monolog_reader_close(r); + return -1; + } + n++; + } + + page->entries = entries; + page->count = n; + monolog_reader_close(r); + *out = page; + return 0; +} + +void monolog_page_free(monolog_page_t *page) +{ + if (!page) { + return; + } + if (page->entries) { + for (uint32_t i = 0; i < page->count; i++) { + free((void *)page->entries[i].date); + free((void *)page->entries[i].channel); + free((void *)page->entries[i].level); + free((void *)page->entries[i].message); + } + free(page->entries); + } + free(page); +} + +uint64_t monolog_page_total_lines(const monolog_page_t *page) +{ + return page ? page->total_lines : 0; +} + +uint32_t monolog_page_count(const monolog_page_t *page) +{ + return page ? page->count : 0; +} + +const monolog_page_entry_t *monolog_page_entry(const monolog_page_t *page, uint32_t index) +{ + if (!page || index >= page->count) { + return NULL; + } + return &page->entries[index]; +} diff --git a/native/monolog_parser/src/simd_scan.c b/native/monolog_parser/src/simd_scan.c new file mode 100644 index 00000000..90a08aac --- /dev/null +++ b/native/monolog_parser/src/simd_scan.c @@ -0,0 +1,176 @@ +/** + * SIMD byte scanners (SSE2 / NEON / scalar fallback). + */ +#include "monolog_parser.h" + +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +# define MONOLOG_X86 1 +# include +#elif defined(__aarch64__) || defined(__ARM_NEON) +# define MONOLOG_NEON 1 +# include +#endif + +const char *monolog_parser_simd(void) +{ +#if defined(MONOLOG_X86) + return "sse2"; +#elif defined(MONOLOG_NEON) + return "neon"; +#else + return "scalar"; +#endif +} + +/* ---- scalar helpers ---------------------------------------------------- */ + +static const char *memchr_scalar(const char *buf, int c, size_t len) +{ + const unsigned char *p = (const unsigned char *)buf; + const unsigned char ch = (unsigned char)c; + for (size_t i = 0; i < len; i++) { + if (p[i] == ch) { + return (const char *)(p + i); + } + } + return NULL; +} + +static const char *memrchr_scalar(const char *buf, int c, size_t len) +{ + const unsigned char *p = (const unsigned char *)buf; + const unsigned char ch = (unsigned char)c; + for (size_t i = len; i > 0; i--) { + if (p[i - 1] == ch) { + return (const char *)(p + i - 1); + } + } + return NULL; +} + +#if defined(MONOLOG_X86) + +/* ---- SSE2 -------------------------------------------------------------- */ + +static const char *memchr_sse2(const char *buf, int c, size_t len) +{ + const unsigned char *p = (const unsigned char *)buf; + const unsigned char ch = (unsigned char)c; + const __m128i needle = _mm_set1_epi8((char)ch); + + while (len >= 16) { + __m128i chunk = _mm_loadu_si128((const __m128i *)p); + __m128i eq = _mm_cmpeq_epi8(chunk, needle); + int mask = _mm_movemask_epi8(eq); + if (mask) { + return (const char *)(p + __builtin_ctz((unsigned)mask)); + } + p += 16; + len -= 16; + } + return memchr_scalar((const char *)p, c, len); +} + +static const char *memrchr_sse2(const char *buf, int c, size_t len) +{ + const unsigned char *base = (const unsigned char *)buf; + const unsigned char ch = (unsigned char)c; + const __m128i needle = _mm_set1_epi8((char)ch); + size_t i = len; + + while (i >= 16) { + i -= 16; + __m128i chunk = _mm_loadu_si128((const __m128i *)(base + i)); + __m128i eq = _mm_cmpeq_epi8(chunk, needle); + int mask = _mm_movemask_epi8(eq); + if (mask) { + return (const char *)(base + i + (31 - __builtin_clz((unsigned)mask))); + } + } + return memrchr_scalar((const char *)base, c, i); +} + +#endif /* MONOLOG_X86 */ + +#if defined(MONOLOG_NEON) + +static const char *memchr_neon(const char *buf, int c, size_t len) +{ + const unsigned char *p = (const unsigned char *)buf; + const uint8x16_t needle = vdupq_n_u8((uint8_t)c); + + while (len >= 16) { + uint8x16_t chunk = vld1q_u8(p); + uint8x16_t eq = vceqq_u8(chunk, needle); + uint8x8_t narrow = vshrn_n_u16(vreinterpretq_u16_u8(eq), 4); + uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(narrow), 0); + if (mask) { + for (int i = 0; i < 16; i++) { + if (p[i] == (unsigned char)c) { + return (const char *)(p + i); + } + } + } + p += 16; + len -= 16; + } + return memchr_scalar((const char *)p, c, len); +} + +static const char *memrchr_neon(const char *buf, int c, size_t len) +{ + return memrchr_scalar(buf, c, len); +} + +#endif /* MONOLOG_NEON */ + +const char *monolog_memchr(const char *buf, int c, size_t len) +{ + if (!buf || len == 0) { + return NULL; + } +#if defined(MONOLOG_X86) + return memchr_sse2(buf, c, len); +#elif defined(MONOLOG_NEON) + return memchr_neon(buf, c, len); +#else + return memchr_scalar(buf, c, len); +#endif +} + +const char *monolog_memrchr(const char *buf, int c, size_t len) +{ + if (!buf || len == 0) { + return NULL; + } +#if defined(MONOLOG_X86) + return memrchr_sse2(buf, c, len); +#elif defined(MONOLOG_NEON) + return memrchr_neon(buf, c, len); +#else + return memrchr_scalar(buf, c, len); +#endif +} + +size_t monolog_find_newlines(const char *buf, size_t len, + size_t *offsets, size_t max_offsets) +{ + size_t count = 0; + const char *p = buf; + const char *end = buf + len; + + while (p < end) { + const char *hit = monolog_memchr(p, '\n', (size_t)(end - p)); + if (!hit) { + break; + } + if (offsets && count < max_offsets) { + offsets[count] = (size_t)(hit - buf); + } + count++; + p = hit + 1; + } + return count; +} diff --git a/native/monolog_parser/tests/test_parser.c b/native/monolog_parser/tests/test_parser.c new file mode 100644 index 00000000..8d3af729 --- /dev/null +++ b/native/monolog_parser/tests/test_parser.c @@ -0,0 +1,320 @@ +/** + * Unit tests for monolog_parser — line parse + reverse file reader. + */ +#define _POSIX_C_SOURCE 200809L + +#include "monolog_parser.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +#define EXPECT(cond, msg) do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \ + failures++; \ + } \ +} while (0) + +static int streq_span(const char *base, int32_t off, int32_t len, const char *expect) +{ + size_t elen = strlen(expect); + if ((size_t)len != elen) { + return 0; + } + return memcmp(base + off, expect, elen) == 0; +} + +static void test_basic(void) +{ + const char *line = + "[2024-03-15T10:30:00.123456+00:00] request.ERROR: Uncaught PHP Exception [] []"; + monolog_fields_t f; + int ok = monolog_parse_line(line, strlen(line), &f); + EXPECT(ok == 1, "basic match"); + EXPECT(f.matched == 1, "matched flag"); + EXPECT(streq_span(line, f.date_off, f.date_len, "2024-03-15T10:30:00.123456+00:00"), "date"); + EXPECT(streq_span(line, f.channel_off, f.channel_len, "request"), "channel"); + EXPECT(streq_span(line, f.level_off, f.level_len, "ERROR"), "level"); + EXPECT(streq_span(line, f.message_off, f.message_len, " Uncaught PHP Exception [] []"), "message"); +} + +static void test_dotted_channel(void) +{ + const char *line = + "[2024-03-15T10:29:59.000000+00:00] doctrine.dbal.INFO: Connecting [] []"; + monolog_fields_t f; + int ok = monolog_parse_line(line, strlen(line), &f); + EXPECT(ok == 1, "dotted channel match"); + EXPECT(streq_span(line, f.channel_off, f.channel_len, "doctrine.dbal"), "channel doctrine.dbal"); + EXPECT(streq_span(line, f.level_off, f.level_len, "INFO"), "level INFO"); +} + +static void test_rightmost_level(void) +{ + const char *line = + "[2024-01-01T00:00:00.000000+00:00] app.INFO: Failed with .ERROR: boom"; + monolog_fields_t f; + int ok = monolog_parse_line(line, strlen(line), &f); + EXPECT(ok == 1, "rightmost match"); + EXPECT(streq_span(line, f.channel_off, f.channel_len, "app.INFO: Failed with "), "greedy channel"); + EXPECT(streq_span(line, f.level_off, f.level_len, "ERROR"), "rightmost level ERROR"); + EXPECT(streq_span(line, f.message_off, f.message_len, " boom"), "message"); +} + +static void test_all_levels(void) +{ + const char *levels[] = { + "DEBUG", "INFO", "NOTICE", "WARNING", "ERROR", "CRITICAL", "ALERT", "EMERGENCY" + }; + char buf[256]; + for (size_t i = 0; i < sizeof(levels) / sizeof(levels[0]); i++) { + snprintf(buf, sizeof(buf), + "[2024-01-01T00:00:00+00:00] ch.%s: msg", levels[i]); + monolog_fields_t f; + int ok = monolog_parse_line(buf, strlen(buf), &f); + EXPECT(ok == 1, levels[i]); + EXPECT(streq_span(buf, f.level_off, f.level_len, levels[i]), levels[i]); + } +} + +static void test_unmatched(void) +{ + const char *line = "not a monolog line"; + monolog_fields_t f; + int ok = monolog_parse_line(line, strlen(line), &f); + EXPECT(ok == 0, "unmatched"); + EXPECT(f.matched == 0, "matched=0"); +} + +static void test_buffer(void) +{ + const char *buf = + "[2024-01-01T00:00:00+00:00] app.INFO: one\n" + "garbage line\n" + "[2024-01-01T00:00:01+00:00] app.ERROR: two\n"; + monolog_fields_t entries[8]; + size_t n = monolog_parse_buffer(buf, strlen(buf), entries, 8); + EXPECT(n == 3, "3 lines"); + EXPECT(entries[0].matched == 1, "line0 match"); + EXPECT(entries[1].matched == 0, "line1 no match"); + EXPECT(entries[2].matched == 1, "line2 match"); + EXPECT(streq_span(buf, entries[0].level_off, entries[0].level_len, "INFO"), "line0 INFO"); + EXPECT(streq_span(buf, entries[2].level_off, entries[2].level_len, "ERROR"), "line2 ERROR"); + EXPECT(streq_span(buf, entries[1].message_off, entries[1].message_len, "garbage line"), "garbage as message"); +} + +static void test_newlines_simd(void) +{ + size_t cap = 1 << 16; + char *buf = malloc(cap); + EXPECT(buf != NULL, "alloc"); + if (!buf) { + return; + } + memset(buf, 'x', cap); + size_t expect = 0; + for (size_t i = 100; i < cap; i += 97) { + buf[i] = '\n'; + expect++; + } + size_t offsets[1024]; + size_t found = monolog_find_newlines(buf, cap, offsets, 1024); + EXPECT(found == expect, "newline count"); + free(buf); +} + +static void test_memrchr(void) +{ + const char *s = "a.b.c.ERROR:x"; + const char *p = monolog_memrchr(s, '.', strlen(s)); + EXPECT(p != NULL && p == s + 5, "memrchr last dot"); +} + +static char *write_temp_log(const char *contents, size_t len) +{ + char tmpl[] = "/tmp/monolog_test_XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + perror("mkstemp"); + return NULL; + } + if (len > 0) { + ssize_t w = write(fd, contents, len); + if (w < 0 || (size_t)w != len) { + perror("write"); + close(fd); + unlink(tmpl); + return NULL; + } + } + close(fd); + return strdup(tmpl); +} + +static void test_reader_yield_order(void) +{ + const char *body = + "[2024-01-01T00:00:00+00:00] app.INFO: first\n" + "[2024-01-01T00:00:01+00:00] app.WARNING: second\n" + "[2024-01-01T00:00:02+00:00] app.ERROR: third\n"; + char *path = write_temp_log(body, strlen(body)); + EXPECT(path != NULL, "temp path"); + if (!path) { + return; + } + + monolog_reader_t *r = monolog_reader_open_backwards(path); + EXPECT(r != NULL, "open"); + if (!r) { + unlink(path); + free(path); + return; + } + + EXPECT(monolog_reader_total_lines(r) == 3, "total 3"); + + monolog_fields_t f; + const char *line = NULL; + int rc; + + rc = monolog_reader_next(r, &f, &line); + EXPECT(rc == 1 && f.matched == 1, "yield 1"); + EXPECT(streq_span(line, f.level_off, f.level_len, "ERROR"), "newest ERROR"); + EXPECT(streq_span(line, f.message_off, f.message_len, " third"), "newest msg"); + + rc = monolog_reader_next(r, &f, &line); + EXPECT(rc == 1 && streq_span(line, f.level_off, f.level_len, "WARNING"), "second WARNING"); + + rc = monolog_reader_next(r, &f, &line); + EXPECT(rc == 1 && streq_span(line, f.level_off, f.level_len, "INFO"), "oldest INFO"); + + rc = monolog_reader_next(r, &f, &line); + EXPECT(rc == 0, "eof"); + + monolog_reader_close(r); + unlink(path); + free(path); +} + +static void test_reader_page_offset_limit(void) +{ + const char *body = + "[2024-01-01T00:00:00+00:00] app.INFO: L0\n" + "[2024-01-01T00:00:01+00:00] app.INFO: L1\n" + "[2024-01-01T00:00:02+00:00] app.INFO: L2\n" + "[2024-01-01T00:00:03+00:00] app.INFO: L3\n" + "[2024-01-01T00:00:04+00:00] app.INFO: L4\n"; + char *path = write_temp_log(body, strlen(body)); + EXPECT(path != NULL, "temp path"); + if (!path) { + return; + } + + monolog_page_t *page = NULL; + int rc = monolog_file_read_backwards(path, 1, 2, &page); + EXPECT(rc == 0 && page != NULL, "page ok"); + if (!page) { + unlink(path); + free(path); + return; + } + + EXPECT(monolog_page_total_lines(page) == 5, "total 5"); + EXPECT(monolog_page_count(page) == 2, "count 2"); + + const monolog_page_entry_t *e0 = monolog_page_entry(page, 0); + const monolog_page_entry_t *e1 = monolog_page_entry(page, 1); + EXPECT(e0 && strcmp(e0->message, " L3") == 0, "page[0]=L3"); + EXPECT(e1 && strcmp(e1->message, " L2") == 0, "page[1]=L2"); + + monolog_page_free(page); + unlink(path); + free(path); +} + +static void test_reader_large_multichunk(void) +{ + size_t lines = 2000; + size_t cap = lines * 120; + char *body = malloc(cap); + EXPECT(body != NULL, "alloc body"); + if (!body) { + return; + } + size_t used = 0; + for (size_t i = 0; i < lines; i++) { + int n = snprintf(body + used, cap - used, + "[2024-01-01T00:00:00+%04zu] ch.INFO: line-%zu-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n", + i, i); + if (n < 0 || (size_t)n >= cap - used) { + break; + } + used += (size_t)n; + } + + char *path = write_temp_log(body, used); + free(body); + EXPECT(path != NULL, "temp large"); + if (!path) { + return; + } + + monolog_page_t *page = NULL; + int rc = monolog_file_read_backwards(path, 0, 5, &page); + EXPECT(rc == 0 && page != NULL, "large page"); + if (page) { + EXPECT(monolog_page_total_lines(page) == lines, "large total"); + EXPECT(monolog_page_count(page) == 5, "large count 5"); + char expect[64]; + snprintf(expect, sizeof(expect), " line-%zu-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", lines - 1); + const monolog_page_entry_t *e0 = monolog_page_entry(page, 0); + EXPECT(e0 && strcmp(e0->message, expect) == 0, "newest message"); + monolog_page_free(page); + } + + monolog_reader_t *r = monolog_reader_open_backwards(path); + EXPECT(r != NULL, "large open"); + if (r) { + uint64_t n = 0; + monolog_fields_t f; + const char *line = NULL; + while (monolog_reader_next(r, &f, &line) == 1) { + n++; + } + EXPECT(n == lines, "yielded all lines"); + monolog_reader_close(r); + } + + unlink(path); + free(path); +} + +int main(void) +{ + printf("monolog_parser %s simd=%s\n", + monolog_parser_version(), monolog_parser_simd()); + + test_basic(); + test_dotted_channel(); + test_rightmost_level(); + test_all_levels(); + test_unmatched(); + test_buffer(); + test_newlines_simd(); + test_memrchr(); + test_reader_yield_order(); + test_reader_page_offset_limit(); + test_reader_large_multichunk(); + + if (failures) { + fprintf(stderr, "\n%d failure(s)\n", failures); + return 1; + } + printf("All tests passed.\n"); + return 0; +} diff --git a/phpstan.neon.dist b/phpstan.neon.dist index b8a6666e..7b689ac4 100644 --- a/phpstan.neon.dist +++ b/phpstan.neon.dist @@ -6,5 +6,23 @@ parameters: ignoreErrors: - message: "#no value type specified in iterable type array#" + # FFI symbols and CData fields are defined by the native CDEF at runtime. + - + message: "#Call to an undefined method FFI::#" + paths: + - src/Components/Log/FfiMonologLogReader.php + - src/Components/Log/FfiMonologParser.php + - + message: "#Access to an undefined property FFI\\\\CData::#" + paths: + - src/Components/Log/FfiMonologLogReader.php + - src/Components/Log/FfiMonologParser.php + - + message: "#Strict comparison using === between FFI\\\\CData and null will always evaluate to false.#" + path: src/Components/Log/FfiMonologLogReader.php + # EntityRepository::search() returns EntitySearchResult on supported 6.6/6.7 releases. + # Its v6.8 hierarchy transition is handled by Shopware core; this plugin only reads entities. + - + message: '#Call to method (getEntities|first)\(\) of deprecated class Shopware\\Core\\Framework\\DataAbstractionLayer\\Search\\EntitySearchResult#' excludePaths: - vendor (?) diff --git a/src/Command/ExtensionChecksumCheckCommand.php b/src/Command/ExtensionChecksumCheckCommand.php index 607ee172..29a2c24d 100644 --- a/src/Command/ExtensionChecksumCheckCommand.php +++ b/src/Command/ExtensionChecksumCheckCommand.php @@ -5,7 +5,6 @@ namespace Frosh\Tools\Command; use Frosh\Tools\Components\ExtensionChecksum\ExtensionFileHashService; -use Shopware\Core\Framework\Adapter\Console\ShopwareStyle; use Shopware\Core\Framework\Context; use Shopware\Core\Framework\DataAbstractionLayer\EntityRepository; use Shopware\Core\Framework\DataAbstractionLayer\Search\Criteria; @@ -17,6 +16,7 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( name: 'frosh:extension:checksum:check', @@ -44,7 +44,7 @@ protected function configure(): void */ protected function execute(InputInterface $input, OutputInterface $output): int { - $io = new ShopwareStyle($input, $output); + $io = new SymfonyStyle($input, $output); $extensions = $this->getExtension((string) $input->getArgument('extension'), $io); if ($extensions->count() < 1) { @@ -93,7 +93,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int return $success ? self::SUCCESS : self::FAILURE; } - private function getExtension(string $name, ShopwareStyle $io): PluginCollection + private function getExtension(string $name, SymfonyStyle $io): PluginCollection { // @phpstan-ignore-next-line $context = method_exists(Context::class, 'createCLIContext') ? Context::createCLIContext() : Context::createDefaultContext(); @@ -124,7 +124,7 @@ private function getExtension(string $name, ShopwareStyle $io): PluginCollection /** * @param string[] $files */ - private function outputFileChanges(ShopwareStyle $io, string $text, array $files): void + private function outputFileChanges(SymfonyStyle $io, string $text, array $files): void { if ($files) { $io->warning($text); diff --git a/src/Command/ExtensionChecksumCreateCommand.php b/src/Command/ExtensionChecksumCreateCommand.php index 8d2d9e26..5f9d5b3a 100644 --- a/src/Command/ExtensionChecksumCreateCommand.php +++ b/src/Command/ExtensionChecksumCreateCommand.php @@ -5,7 +5,6 @@ namespace Frosh\Tools\Command; use Frosh\Tools\Components\ExtensionChecksum\ExtensionFileHashService; -use Shopware\Core\Framework\Adapter\Console\ShopwareStyle; use Shopware\Core\Framework\Context; use Shopware\Core\Framework\DataAbstractionLayer\Entity; use Shopware\Core\Framework\DataAbstractionLayer\EntityRepository; @@ -18,6 +17,7 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( name: 'frosh:extension:checksum:create', @@ -45,7 +45,7 @@ protected function configure(): void */ protected function execute(InputInterface $input, OutputInterface $output): int { - $io = new ShopwareStyle($input, $output); + $io = new SymfonyStyle($input, $output); // @phpstan-ignore-next-line $context = method_exists(Context::class, 'createCLIContext') ? Context::createCLIContext() : Context::createDefaultContext(); diff --git a/src/Command/MonitorCommand.php b/src/Command/MonitorCommand.php index b0fd959d..71006e10 100644 --- a/src/Command/MonitorCommand.php +++ b/src/Command/MonitorCommand.php @@ -10,7 +10,6 @@ use Frosh\Tools\Components\Health\SettingsResult; use Shopware\Core\Content\Mail\Service\AbstractMailSender; use Shopware\Core\Content\Mail\Service\MailSender; -use Shopware\Core\Framework\Adapter\Console\ShopwareStyle; use Shopware\Core\System\SystemConfig\SystemConfigService; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -18,6 +17,7 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; use Symfony\Component\DependencyInjection\Attribute\Autowire; use Symfony\Component\Mime\Email; @@ -46,7 +46,7 @@ protected function configure(): void protected function execute(InputInterface $input, OutputInterface $output): int { - $io = new ShopwareStyle($input, $output); + $io = new SymfonyStyle($input, $output); if ($input->getArgument('sales-channel')) { $io->warning('The sales channel argument is deprecated and has no effect. It will be removed in a future release.'); diff --git a/src/Components/Log/FfiMonologLogReader.php b/src/Components/Log/FfiMonologLogReader.php new file mode 100644 index 00000000..4aeab286 --- /dev/null +++ b/src/Components/Log/FfiMonologLogReader.php @@ -0,0 +1,196 @@ +ffi = $ffi; + } + + /** + * @throws \Throwable + */ + public static function create(?string $libraryPath = null): self + { + if (!\extension_loaded('ffi') || !class_exists(\FFI::class, false)) { + throw new \RuntimeException('ext-ffi is not available'); + } + + $path = $libraryPath ?? NativeLibraryLocator::locate(); + if ($path === null || !is_readable($path)) { + throw new \RuntimeException( + 'libmonolog_parser not found for ' . NativeLibraryLocator::describe() + ); + } + + /** @var \FFI $ffi */ + $ffi = \FFI::cdef(self::CDEF, $path); + + return new self($ffi); + } + + public static function tryCreate(?string $libraryPath = null): ?self + { + try { + return self::create($libraryPath); + } catch (\Throwable) { + return null; + } + } + + public function readBackwards(string $filePath, int $offset, int $limit): array + { + /** @var \FFI\CData $pagePtr */ + $pagePtr = $this->ffi->new('monolog_page_t*'); + $rc = $this->ffi->monolog_file_read_backwards( + $filePath, + (int) max(0, $offset), + (int) max(0, $limit), + \FFI::addr($pagePtr), + ); + + if ($rc !== 0 || $pagePtr === null) { + throw new \RuntimeException(\sprintf('monolog_file_read_backwards failed for %s', $filePath)); + } + + try { + $total = (int) $this->ffi->monolog_page_total_lines($pagePtr); + $count = (int) $this->ffi->monolog_page_count($pagePtr); + $entries = []; + + for ($i = 0; $i < $count; ++$i) { + $entry = $this->ffi->monolog_page_entry($pagePtr, $i); + if ($entry === null) { + continue; + } + $entries[] = [ + 'date' => (string) $entry->date, + 'channel' => (string) $entry->channel, + 'level' => (string) $entry->level, + 'message' => (string) $entry->message, + ]; + } + + return [ + 'entries' => $entries, + 'total' => $total, + ]; + } finally { + $this->ffi->monolog_page_free($pagePtr); + } + } + + public function yieldBackwards(string $filePath): \Generator + { + $reader = $this->ffi->monolog_reader_open_backwards($filePath); + if ($reader === null) { + throw new \RuntimeException(\sprintf('Cannot open log file: %s', $filePath)); + } + + try { + /** @var \FFI\CData $fields */ + $fields = $this->ffi->new('monolog_fields_t'); + /** @var \FFI\CData $linePtr */ + $linePtr = $this->ffi->new('const char*'); + + while (true) { + $rc = $this->ffi->monolog_reader_next($reader, \FFI::addr($fields), \FFI::addr($linePtr)); + if ($rc === 0) { + break; + } + if ($rc < 0) { + $err = $this->ffi->monolog_reader_error($reader); + throw new \RuntimeException('monolog_reader_next: ' . ($err ?? 'unknown error')); + } + + $line = \FFI::string($linePtr); + if ($fields->matched === 1) { + yield [ + 'date' => substr($line, $fields->date_off, $fields->date_len), + 'channel' => substr($line, $fields->channel_off, $fields->channel_len), + 'level' => substr($line, $fields->level_off, $fields->level_len), + 'message' => substr($line, $fields->message_off, $fields->message_len), + ]; + } else { + yield [ + 'date' => 'unknown', + 'channel' => 'unknown', + 'level' => 'unknown', + 'message' => $line, + ]; + } + } + } finally { + $this->ffi->monolog_reader_close($reader); + } + } + + public function backend(): string + { + return 'ffi'; + } + + public function simd(): string + { + return (string) $this->ffi->monolog_parser_simd(); + } + + public function version(): string + { + return (string) $this->ffi->monolog_parser_version(); + } +} diff --git a/src/Components/Log/FfiMonologParser.php b/src/Components/Log/FfiMonologParser.php new file mode 100644 index 00000000..1622a299 --- /dev/null +++ b/src/Components/Log/FfiMonologParser.php @@ -0,0 +1,123 @@ +ffi = $ffi; + } + + /** + * @throws \Throwable when FFI/.so cannot be used + */ + public static function create(?string $libraryPath = null): self + { + if (!\extension_loaded('ffi')) { + throw new \RuntimeException('ext-ffi is not loaded'); + } + + if (!class_exists(\FFI::class, false)) { + throw new \RuntimeException('FFI class unavailable'); + } + + $path = $libraryPath ?? NativeLibraryLocator::locate(); + if ($path === null || !is_readable($path)) { + throw new \RuntimeException( + 'libmonolog_parser not found for ' . NativeLibraryLocator::describe() + . ' — ship prebuilds under native/lib/ or set MONOLOG_PARSER_LIB' + ); + } + + /** @var \FFI $ffi */ + $ffi = \FFI::cdef(self::CDEF, $path); + + return new self($ffi); + } + + public static function tryCreate(?string $libraryPath = null): ?self + { + try { + return self::create($libraryPath); + } catch (\Throwable) { + return null; + } + } + + public function parse(string $line): array + { + $len = \strlen($line); + /** @var \FFI\CData $fields */ + $fields = $this->ffi->new('monolog_fields_t'); + $ok = $this->ffi->monolog_parse_line($line, $len, \FFI::addr($fields)); + + if ($ok !== 1 || $fields->matched !== 1) { + return [ + 'message' => $line, + 'channel' => 'unknown', + 'date' => 'unknown', + 'level' => 'unknown', + ]; + } + + return [ + 'date' => substr($line, $fields->date_off, $fields->date_len), + 'channel' => substr($line, $fields->channel_off, $fields->channel_len), + 'level' => substr($line, $fields->level_off, $fields->level_len), + 'message' => substr($line, $fields->message_off, $fields->message_len), + ]; + } + + public function backend(): string + { + return 'ffi'; + } + + public function simd(): string + { + return (string) $this->ffi->monolog_parser_simd(); + } + + public function version(): string + { + return (string) $this->ffi->monolog_parser_version(); + } + + /** + * @deprecated use NativeLibraryLocator::locate() + */ + public static function detectLibraryPath(): ?string + { + return NativeLibraryLocator::locate(); + } +} diff --git a/src/Components/Log/MonologLineParserFactory.php b/src/Components/Log/MonologLineParserFactory.php new file mode 100644 index 00000000..67ea6572 --- /dev/null +++ b/src/Components/Log/MonologLineParserFactory.php @@ -0,0 +1,42 @@ +, total: int} + */ +interface MonologLogReaderInterface +{ + /** + * Read a page of log lines from the end of the file. + * + * offset 0 = newest line. Lines are already parsed. + * + * @return MonologPage + */ + public function readBackwards(string $filePath, int $offset, int $limit): array; + + /** + * Stream parsed entries newest-first (PHP generator / "yield"). + * + * @return \Generator + */ + public function yieldBackwards(string $filePath): \Generator; + + /** + * Implementation id: "ffi" or "preg". + */ + public function backend(): string; +} diff --git a/src/Components/Log/NativeLibraryLocator.php b/src/Components/Log/NativeLibraryLocator.php new file mode 100644 index 00000000..9656d4ba --- /dev/null +++ b/src/Components/Log/NativeLibraryLocator.php @@ -0,0 +1,239 @@ + 'darwin', + 'Linux' => 'linux', + default => strtolower($osFamily), + }; + + $arch = match (true) { + \in_array($rawArch, ['x86_64', 'amd64', 'x64'], true) => 'x86_64', + \in_array($rawArch, ['aarch64', 'arm64'], true) => 'arm64', + default => $rawArch, + }; + + $ext = match ($os) { + 'darwin' => 'dylib', + 'windows' => 'dll', + default => 'so', + }; + + // libc only meaningful on Linux (gnu vs musl). Darwin/Windows → "n/a". + $libc = $os === 'linux' ? (self::isMusl() ? 'musl' : 'gnu') : 'n/a'; + + $triple = $os === 'linux' && $libc === 'musl' + ? \sprintf('linux-musl-%s', $arch) + : \sprintf('%s-%s', $os, $arch); + + return [ + 'os' => $os, + 'arch' => $arch, + 'libc' => $libc, + 'ext' => $ext, + 'triple' => $triple, + ]; + } + + /** + * Detect musl (Alpine, void, etc.). + * + * Order: + * 1. MONOLOG_PARSER_LIBC=musl|gnu|glibc + * 2. Presence of /lib/ld-musl-*.so.1 + * 3. ldd on PHP_BINARY mentions musl + * 4. phpinfo() / PHP_OS extras (last resort) + */ + public static function isMusl(): bool + { + static $cached = null; + if ($cached !== null) { + return $cached; + } + + $env = getenv('MONOLOG_PARSER_LIBC'); + if (\is_string($env) && $env !== '') { + $v = strtolower($env); + if (\in_array($v, ['musl'], true)) { + return $cached = true; + } + if (\in_array($v, ['gnu', 'glibc', 'gcompat'], true)) { + return $cached = false; + } + } + + if (\PHP_OS_FAMILY !== 'Linux') { + return $cached = false; + } + + // Canonical musl dynamic linker paths (Alpine multi-arch) + $loaders = [ + '/lib/ld-musl-x86_64.so.1', + '/lib/ld-musl-aarch64.so.1', + '/lib/ld-musl-armhf.so.1', + '/lib/ld-musl-i386.so.1', + '/lib/ld-musl-riscv64.so.1', + ]; + foreach ($loaders as $loader) { + if (is_readable($loader)) { + return $cached = true; + } + } + + // Glob fallback (some layouts) + foreach (glob('/lib/ld-musl-*.so.*') ?: [] as $loader) { + if (is_readable($loader)) { + return $cached = true; + } + } + + // Inspect what PHP itself is linked against + if (is_readable(\PHP_BINARY)) { + $ldd = self::safeLdd(\PHP_BINARY); + if ($ldd !== null) { + if (str_contains($ldd, 'musl')) { + return $cached = true; + } + // Explicit glibc markers + if (str_contains($ldd, 'libc.so.6') || str_contains($ldd, 'ld-linux')) { + return $cached = false; + } + } + } + + return $cached = false; + } + + /** + * Preferred basenames, most specific first. + * + * @return list + */ + public static function candidateBasenames(): array + { + $p = self::platform(); + $names = []; + + if ($p['os'] === 'linux' && $p['libc'] === 'musl') { + // musl hosts must not load a glibc .so (hard ELF failure) + $names[] = \sprintf('libmonolog_parser-linux-musl-%s.so', $p['arch']); + if ($p['arch'] === 'arm64') { + $names[] = 'libmonolog_parser-linux-musl-aarch64.so'; + } + } elseif ($p['os'] === 'linux') { + $names[] = \sprintf('libmonolog_parser-linux-%s.so', $p['arch']); + if ($p['arch'] === 'arm64') { + $names[] = 'libmonolog_parser-linux-aarch64.so'; + } + // Do not fall back to musl on glibc — wrong ABI + } else { + $names[] = \sprintf('libmonolog_parser-%s-%s.%s', $p['os'], $p['arch'], $p['ext']); + } + + // Legacy unprefixed names (local `make install`) + $names[] = 'libmonolog_parser.' . $p['ext']; + + return $names; + } + + /** + * Human-readable platform string for diagnostics. + */ + public static function describe(): string + { + $p = self::platform(); + $candidates = self::candidateBasenames(); + + return \sprintf( + '%s libc=%s (php_uname=%s, looking for %s)', + $p['triple'], + $p['libc'], + php_uname('m'), + $candidates[0] ?? '?' + ); + } + + private static function safeLdd(string $binary): ?string + { + // Avoid shell if disabled; skip in restricted SAPIs + if (!\function_exists('shell_exec')) { + return null; + } + + // open_basedir / disabled functions may still kill this + try { + $cmd = 'ldd ' . escapeshellarg($binary) . ' 2>&1'; + $out = @shell_exec($cmd); + } catch (\Throwable) { + return null; + } + + return \is_string($out) ? $out : null; + } +} diff --git a/src/Components/Log/PregMonologLogReader.php b/src/Components/Log/PregMonologLogReader.php new file mode 100644 index 00000000..073edf7f --- /dev/null +++ b/src/Components/Log/PregMonologLogReader.php @@ -0,0 +1,49 @@ +seek(\PHP_INT_MAX); + $total = $file->key(); + + $entries = []; + $reader = new \LimitIterator(LineReader::readLinesBackwards($filePath), $offset, $limit); + /** @var string $line */ + foreach ($reader as $line) { + $entries[] = $this->parser->parse($line); + } + + return [ + 'entries' => $entries, + 'total' => $total, + ]; + } + + public function yieldBackwards(string $filePath): \Generator + { + foreach (LineReader::readLinesBackwards($filePath) as $line) { + yield $this->parser->parse($line); + } + } + + public function backend(): string + { + return 'preg'; + } +} diff --git a/src/Components/Log/PregMonologParser.php b/src/Components/Log/PregMonologParser.php new file mode 100644 index 00000000..0d9f4c2f --- /dev/null +++ b/src/Components/Log/PregMonologParser.php @@ -0,0 +1,40 @@ +.*)] (?.*)\.(?(DEBUG|INFO|NOTICE|WARNING|ERROR|CRITICAL|ALERT|EMERGENCY)):(?.*)/m'; + + public function parse(string $line): array + { + if (preg_match(self::LINE_MATCH, $line, $matches) !== 1) { + return [ + 'message' => $line, + 'channel' => 'unknown', + 'date' => 'unknown', + 'level' => 'unknown', + ]; + } + + return [ + 'message' => $matches['message'], + 'channel' => $matches['channel'], + 'date' => $matches['date'], + 'level' => $matches['level'], + ]; + } + + public function backend(): string + { + return 'preg'; + } +} diff --git a/src/Controller/LogController.php b/src/Controller/LogController.php index ba18402d..d55b3c90 100644 --- a/src/Controller/LogController.php +++ b/src/Controller/LogController.php @@ -4,7 +4,7 @@ namespace Frosh\Tools\Controller; -use Frosh\Tools\Components\LineReader; +use Frosh\Tools\Components\Log\MonologLogReaderInterface; use Shopware\Core\Framework\Routing\RoutingException; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\DependencyInjection\Attribute\Autowire; @@ -17,14 +17,12 @@ #[Route(path: '/api/_action/frosh-tools', defaults: ['_routeScope' => ['api'], '_acl' => ['frosh_tools:read']])] class LogController extends AbstractController { - // https://regex101.com/r/bp4YYL/1 - private const LINE_MATCH = '/\[(?.*)] (?.*)\.(?(DEBUG|INFO|NOTICE|WARNING|ERROR|CRITICAL|ALERT|EMERGENCY)):(?.*)/m'; - private readonly string $logDir; public function __construct( #[Autowire(param: 'kernel.logs_dir')] string $logDir, + private readonly MonologLogReaderInterface $monologLogReader, ) { $this->logDir = rtrim($logDir, '/') . '/'; } @@ -42,36 +40,14 @@ public function getLog(Request $request): Response $offset = $request->query->getInt('offset'); $limit = $request->query->getInt('limit', 20); - $lineGenerator = LineReader::readLinesBackwards($filePath); - $file = new \SplFileObject($filePath, 'r'); - $file->seek(\PHP_INT_MAX); - - $reader = new \LimitIterator($lineGenerator, $offset, $limit); - - $result = []; - - /** @var string $item */ - foreach ($reader as $item) { - if (preg_match(self::LINE_MATCH, $item, $matches) !== 1) { - $result[] = [ - 'message' => $item, - 'channel' => 'unknown', - 'date' => 'unknown', - 'level' => 'unknown', - ]; - - continue; - } - - $result[] = [ - 'message' => $matches['message'], - 'channel' => $matches['channel'], - 'date' => $matches['date'], - 'level' => $matches['level'], - ]; - } + // Native path: open file in C, reverse-scan + SIMD parse, return page. + // Fallback path: PHP LineReader + preg_match (same response shape). + $page = $this->monologLogReader->readBackwards($filePath, $offset, $limit); - return new JsonResponse($result, Response::HTTP_OK, ['file-size' => $file->key()]); + return new JsonResponse($page['entries'], Response::HTTP_OK, [ + 'file-size' => $page['total'], + 'x-monolog-parser' => $this->monologLogReader->backend(), + ]); } private function getFilePathByBag(Request $request): string diff --git a/src/Resources/config/services.php b/src/Resources/config/services.php index 7e71980a..c754015a 100644 --- a/src/Resources/config/services.php +++ b/src/Resources/config/services.php @@ -2,6 +2,10 @@ declare(strict_types=1); +use Frosh\Tools\Components\Log\MonologLineParserFactory; +use Frosh\Tools\Components\Log\MonologLineParserInterface; +use Frosh\Tools\Components\Log\MonologLogReaderFactory; +use Frosh\Tools\Components\Log\MonologLogReaderInterface; use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; return static function (ContainerConfigurator $container): void { @@ -12,4 +16,10 @@ $services->load('Frosh\Tools\\', '../../') ->exclude('../../{DependencyInjection,Resources,FroshTools.php}'); + + $services->set(MonologLineParserInterface::class) + ->factory([MonologLineParserFactory::class, 'create']); + + $services->set(MonologLogReaderInterface::class) + ->factory([MonologLogReaderFactory::class, 'create']); }; diff --git a/tests/Unit/Log/PregMonologParserTest.php b/tests/Unit/Log/PregMonologParserTest.php new file mode 100644 index 00000000..ff04356c --- /dev/null +++ b/tests/Unit/Log/PregMonologParserTest.php @@ -0,0 +1,69 @@ +parser = new PregMonologParser(); + } + + public function testParsesStandardLine(): void + { + $line = '[2024-03-15T10:30:00.123456+00:00] request.ERROR: Uncaught PHP Exception [] []'; + $entry = $this->parser->parse($line); + + static::assertSame('2024-03-15T10:30:00.123456+00:00', $entry['date']); + static::assertSame('request', $entry['channel']); + static::assertSame('ERROR', $entry['level']); + static::assertSame(' Uncaught PHP Exception [] []', $entry['message']); + } + + public function testDottedChannel(): void + { + $line = '[2024-03-15T10:29:59.000000+00:00] doctrine.dbal.INFO: Connecting [] []'; + $entry = $this->parser->parse($line); + + static::assertSame('doctrine.dbal', $entry['channel']); + static::assertSame('INFO', $entry['level']); + } + + public function testUnmatchedFallsBackToUnknown(): void + { + $entry = $this->parser->parse('not a log line'); + + static::assertSame('unknown', $entry['channel']); + static::assertSame('unknown', $entry['level']); + static::assertSame('unknown', $entry['date']); + static::assertSame('not a log line', $entry['message']); + } + + /** + * @return iterable + */ + public static function levelsProvider(): iterable + { + foreach (['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL', 'ALERT', 'EMERGENCY'] as $level) { + yield $level => [$level]; + } + } + + #[DataProvider('levelsProvider')] + public function testAllLevels(string $level): void + { + $line = \sprintf('[2024-01-01T00:00:00+00:00] ch.%s: msg', $level); + $entry = $this->parser->parse($line); + static::assertSame($level, $entry['level']); + } +}