Adding macOS unified logging source to collectors

changelog/unreleased/pr-26542.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Add macOS Unified Logging Collector source"
+
+pulls = ["26542"]
+

graylog2-server/src/main/java/org/graylog/collectors/CollectorsModule.java

@@ -22,10 +22,12 @@
 import org.graylog.collectors.cloud.CloudCollectorIngestService;
 import org.graylog.collectors.config.receiver.FilelogReceiverConfig;
 import org.graylog.collectors.config.receiver.JournaldReceiverConfig;
+import org.graylog.collectors.config.receiver.MacOSUnifiedLoggingReceiverConfig;
 import org.graylog.collectors.config.receiver.WindowsEventLogReceiverConfig;
 import org.graylog.collectors.db.FileSourceConfig;
 import org.graylog.collectors.db.FleetDTO;
 import org.graylog.collectors.db.JournaldSourceConfig;
+import org.graylog.collectors.db.MacOSUnifiedLoggingSourceConfig;
 import org.graylog.collectors.db.WindowsEventLogSourceConfig;
 import org.graylog.collectors.indexer.CollectorLogsIndexTemplateProvider;
 import org.graylog.collectors.input.CollectorIngestCodec;
@@ -37,6 +39,7 @@
 import org.graylog.collectors.input.processor.FilelogRecordProcessor;
 import org.graylog.collectors.input.processor.JournaldRecordProcessor;
 import org.graylog.collectors.input.processor.LogRecordProcessor;
+import org.graylog.collectors.input.processor.MacOSUnifiedLoggingRecordProcessor;
 import org.graylog.collectors.input.processor.WindowsEventLogRecordProcessor;
 import org.graylog.collectors.input.transport.CollectorIngestHttpTransport;
 import org.graylog.collectors.opamp.OpAmpModule;
@@ -97,6 +100,7 @@ protected void configure() {
         logRecordProcessorBinder.addBinding(FilelogReceiverConfig.RECEIVER_TYPE).to(FilelogRecordProcessor.class);
         logRecordProcessorBinder.addBinding(JournaldReceiverConfig.RECEIVER_TYPE).to(JournaldRecordProcessor.class);
         logRecordProcessorBinder.addBinding(WindowsEventLogReceiverConfig.RECEIVER_TYPE).to(WindowsEventLogRecordProcessor.class);
+        logRecordProcessorBinder.addBinding(MacOSUnifiedLoggingReceiverConfig.RECEIVER_TYPE).to(MacOSUnifiedLoggingRecordProcessor.class);
         logRecordProcessorBinder.addBinding(CollectorLogRecordProcessor.RECEIVER_TYPE).to(CollectorLogRecordProcessor.class);
 
         if (otlpDumpEnabled) {
@@ -143,6 +147,7 @@ protected void configure() {
         registerJacksonSubtype(FileSourceConfig.class);
         registerJacksonSubtype(JournaldSourceConfig.class);
         registerJacksonSubtype(WindowsEventLogSourceConfig.class);
+        registerJacksonSubtype(MacOSUnifiedLoggingSourceConfig.class);
 
         final var indexTemplateProviderBinder = MapBinder.newMapBinder(binder(), String.class,
                 IndexTemplateProvider.class);

graylog2-server/src/main/java/org/graylog/collectors/config/receiver/MacOSUnifiedLoggingReceiverConfig.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.collectors.config.receiver;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.auto.value.AutoValue;
+import jakarta.annotation.Nullable;
+import org.graylog.collectors.CollectorOSType;
+import org.graylog.collectors.config.GoDurationSerializer;
+import org.graylog.collectors.config.extension.FileStorageExtensionConfig;
+
+import java.time.Duration;
+import java.util.EnumSet;
+
+import static org.graylog2.shared.utilities.StringUtils.f;
+
+/**
+ * OTel collector macOS Unified Logging Receiver configuration.
+ * <p>
+ * Graylog only models the live-collection subset of the upstream receiver's options. The upstream
+ * {@code archive_path} (one-shot collection from a static {@code .logarchive}) and {@code end_time}
+ * (a bounded, finite collection window) options are intentionally omitted: the collector is meant
+ * for continuous live tailing, so those would only ever produce one-shot behavior.
+ *
+ * @see <a href="https://github.com/Graylog2/collector/tree/main/receiver/macosunifiedloggingreceiver">macOS Unified Logging Receiver</a>
+ */
+@AutoValue
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public abstract class MacOSUnifiedLoggingReceiverConfig implements CollectorReceiverConfig {
+    public static final String RECEIVER_TYPE = "macos_unified_logging";
+
+    public String type() {
+        return RECEIVER_TYPE;
+    }
+
+    @Override
+    public EnumSet<CollectorOSType> osSupport() {
+        return EnumSet.of(CollectorOSType.MACOS);
+    }
+
+    @Nullable
+    @JsonProperty("predicate")
+    public abstract String predicate();
+
+    @JsonProperty("max_poll_interval")
+    @JsonSerialize(using = GoDurationSerializer.class)
+    public abstract Duration maxPollInterval();
+
+    @JsonProperty("max_log_age")
+    @JsonSerialize(using = GoDurationSerializer.class)
+    public abstract Duration maxLogAge();
+
+    @JsonProperty("format")
+    public String format() {
+        return "ndjson";
+    }
+
+    // Persists the live-mode forward cursor; required for cursor tracking and de-duplication.
+    @JsonProperty("storage")
+    public abstract String storage();
+
+
+    public static Builder builder(String id) {
+        return new AutoValue_MacOSUnifiedLoggingReceiverConfig.Builder()
+                .name(f("macos_unified_logging/%s", id))
+                .maxPollInterval(Duration.ofSeconds(30))
+                .maxLogAge(Duration.ofHours(24))
+                .storage(FileStorageExtensionConfig.defaultInstance().name());
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder {
+        public abstract Builder name(String name);
+
+        public abstract Builder predicate(@Nullable String predicate);
+
+        public abstract Builder maxPollInterval(Duration maxPollInterval);
+
+        public abstract Builder maxLogAge(Duration maxLogAge);
+
+        public abstract Builder storage(String storage);
+
+        public abstract MacOSUnifiedLoggingReceiverConfig build();
+    }
+}

graylog2-server/src/main/java/org/graylog/collectors/db/MacOSUnifiedLoggingSourceConfig.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.collectors.db;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Strings;
+import jakarta.annotation.Nullable;
+import org.graylog.collectors.config.receiver.CollectorReceiverConfig;
+import org.graylog.collectors.config.receiver.MacOSUnifiedLoggingReceiverConfig;
+
+import java.time.Duration;
+import java.util.Optional;
+
+@AutoValue
+@JsonTypeName(MacOSUnifiedLoggingSourceConfig.TYPE_NAME)
+@JsonDeserialize(builder = MacOSUnifiedLoggingSourceConfig.Builder.class)
+public abstract class MacOSUnifiedLoggingSourceConfig implements SourceConfig {
+    public static final String TYPE_NAME = "macos_unified_logging";
+
+    @Override
+    @JsonProperty(TYPE_FIELD)
+    public abstract String type();
+
+    @Nullable
+    @JsonProperty("predicate")
+    public abstract String predicate();
+
+    @JsonProperty("max_poll_interval")
+    public abstract Duration maxPollInterval();
+
+    @JsonProperty("max_log_age")
+    public abstract Duration maxLogAge();
+
+    public static Builder builder() {
+        return Builder.create();
+    }
+
+    @Override
+    public void validate() {
+        // empty predicates are valid
+        if (maxPollInterval().isZero() || maxPollInterval().isNegative()) {
+            throw new IllegalArgumentException("max_poll_interval must be positive");
+        }
+        if (maxLogAge().isNegative()) {
+            throw new IllegalArgumentException("max_log_age must be zero or positive");
+        }
+    }
+
+    @Override
+    public Optional<CollectorReceiverConfig> toReceiverConfig(String id) {
+        final var builder = MacOSUnifiedLoggingReceiverConfig.builder(id)
+                .maxLogAge(maxLogAge())
+                .maxPollInterval(maxPollInterval());
+        if (predicate() != null) {
+            builder.predicate(Strings.emptyToNull(predicate()));
+        }
+        return Optional.of(builder.build());
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder {
+        @JsonCreator
+        public static Builder create() {
+            return new AutoValue_MacOSUnifiedLoggingSourceConfig.Builder()
+                    .type(TYPE_NAME);
+        }
+
+        @JsonProperty(TYPE_FIELD)
+        public abstract Builder type(String type);
+
+        @JsonProperty("predicate")
+        public abstract Builder predicate(@Nullable String predicate);
+
+        @JsonProperty("max_poll_interval")
+        public abstract Builder maxPollInterval(Duration maxPollInterval);
+
+        @JsonProperty("max_log_age")
+        public abstract Builder maxLogAge(Duration maxLogAge);
+
+        public abstract MacOSUnifiedLoggingSourceConfig build();
+    }
+}

graylog2-server/src/main/java/org/graylog/collectors/input/processor/MacOSUnifiedLoggingRecordProcessor.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.collectors.input.processor;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang.StringUtils;
+import org.graylog.inputs.otel.OTelJournal;
+import org.graylog.schema.EventFields;
+import org.graylog.schema.ProcessFields;
+import org.graylog.schema.UserFields;
+import org.graylog.schema.VendorFields;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Processes macOS unified logging records into GIM fields.
+ * <p>
+ * The fixed receiver emits a plain-text body (the human-readable message), native OTel severity
+ * and timestamp — all handled by {@link org.graylog.collectors.input.CollectorIngestCodec} — plus
+ * structured {@code macos.*} attributes. This processor maps only those attributes: well-fitting
+ * ones to GIM fields, and macOS-specific identifiers (boot session, activity/trace IDs, sender
+ * image) preserved under a {@code macos_*} prefix.
+ *
+ * @see <a href="https://developer.apple.com/documentation/os/logging">Apple Unified Logging</a>
+ */
+public class MacOSUnifiedLoggingRecordProcessor implements LogRecordProcessor {
+
+    @Override
+    public Map<String, Object> process(OTelJournal.Log log) {
+        final Map<String, Object> result = new HashMap<>();
+        String processImagePath = null;
+
+        for (final var attr : log.getLogRecord().getAttributesList()) {
+            final var value = attr.getValue();
+            switch (attr.getKey()) {
+                // Mapped to GIM fields.
+                case "macos.subsystem" -> putStr(result, EventFields.EVENT_SOURCE, value.getStringValue());
+                case "macos.category" -> putStr(result, VendorFields.VENDOR_SUBTYPE, value.getStringValue());
+                case "macos.eventType" -> putStr(result, VendorFields.VENDOR_EVENT_TYPE, value.getStringValue());
+                case "macos.formatString" -> putStr(result, VendorFields.VENDOR_EVENT_DESCRIPTION, value.getStringValue());
+                case "macos.processImagePath" -> {
+                    final var path = Strings.emptyToNull(value.getStringValue());
+                    if (path != null) {
+                        processImagePath = path;
+                        result.put(ProcessFields.PROCESS_PATH, path);
+                    }
+                }
+                case "macos.processID" -> result.put(ProcessFields.PROCESS_ID, Long.toString(value.getIntValue()));
+                case "macos.userID" -> result.put(UserFields.USER_ID, Long.toString(value.getIntValue()));
+                // macOS-specific identifiers preserved under a macos_* prefix.
+                case "macos.threadID" -> result.put("macos_thread_id", value.getIntValue());
+                case "macos.bootUUID" -> putStr(result, "macos_boot_uuid", value.getStringValue());
+                case "macos.machTimestamp" -> result.put("macos_mach_timestamp", value.getIntValue());
+                case "macos.traceID" -> result.put("macos_trace_id", value.getIntValue());
+                case "macos.activityIdentifier" -> result.put("macos_activity_id", value.getIntValue());
+                case "macos.parentActivityIdentifier" -> result.put("macos_parent_activity_id", value.getIntValue());
+                case "macos.creatorActivityID" -> result.put("macos_creator_activity_id", value.getIntValue());
+                case "macos.processImageUUID" -> putStr(result, "macos_process_image_uuid", value.getStringValue());
+                case "macos.senderImagePath" -> putStr(result, "macos_sender_image_path", value.getStringValue());
+                case "macos.senderImageUUID" -> putStr(result, "macos_sender_image_uuid", value.getStringValue());
+                case "macos.senderProgramCounter" -> result.put("macos_sender_program_counter", value.getIntValue());
+                default -> {
+                    // Ignore non-macos attributes.
+                }
+            }
+        }
+
+        // Derive the process name from the executable path.
+        if (processImagePath != null) {
+            Path path = Paths.get(processImagePath);
+            final var fileName = path.getFileName();
+            if (fileName != null) {
+                result.put(ProcessFields.PROCESS_NAME, fileName.toString());
+            }
+        }
+
+        return result;
+    }
+
+    private static void putStr(Map<String, Object> target, String field, String value) {
+        if (StringUtils.isNotBlank(value)) {
+            target.put(field, value);
+        }
+    }
+}