diff --git a/bigquery/client.go b/bigquery/client.go index c6aa534..2a6ae12 100644 --- a/bigquery/client.go +++ b/bigquery/client.go @@ -30,26 +30,6 @@ type BigQueryAdapter struct { cancel context.CancelFunc } -func (bq *BigQueryConfig) Validate() error { - if bq.ProjectId == "" { - return errors.New("missing project_id") - } - // this will usually be th same as projectID but could be different if using outside project dataset such as a public data set - if bq.BigQueryProject == "" { - return errors.New("missing bigquery project name") - } - if bq.DatasetName == "" { - return errors.New("missing dataset_name") - } - if bq.TableName == "" { - return errors.New("missing table_name") - } - if bq.SqlQuery == "" { - return errors.New("missing sql query") - } - return nil -} - func NewBigQueryAdapter(ctx context.Context, conf BigQueryConfig) (*BigQueryAdapter, chan struct{}, error) { // Create bq cancellable context ctx, cancel := context.WithCancel(context.Background()) diff --git a/bigquery/conf.go b/bigquery/conf.go index 56037e9..07c225c 100644 --- a/bigquery/conf.go +++ b/bigquery/conf.go @@ -1,6 +1,8 @@ package usp_bigquery import ( + "errors" + "github.com/refractionPOINT/go-uspclient" ) @@ -15,3 +17,33 @@ type BigQueryConfig struct { QueryInterval string `json:"query_interval" yaml:"query_interval"` IsOneTimeLoad bool `json:"is_one_time_load" yaml:"is_one_time_load"` } + +// Validate validates the BigQuery adapter configuration. +// +// Parameters: +// +// None +// +// Returns: +// +// error - Returns nil if validation passes, or an error describing the validation failure. +func (c *BigQueryConfig) Validate() error { + if c.ProjectId == "" { + return errors.New("missing project_id") + } + // BigQueryProject will usually be the same as ProjectId but could be different + // if using outside project dataset such as a public data set. + if c.BigQueryProject == "" { + return errors.New("missing bigquery_project") + } + if c.DatasetName == "" { + return errors.New("missing dataset_name") + } + if c.TableName == "" { + return errors.New("missing table_name") + } + if c.SqlQuery == "" { + return errors.New("missing sql_query") + } + return nil +} diff --git a/containers/general/tool.go b/containers/general/tool.go index 4c7af0b..1c67b15 100755 --- a/containers/general/tool.go +++ b/containers/general/tool.go @@ -69,6 +69,11 @@ type USPClient interface { Close() error } +// ConfigValidator is implemented by adapter configs that support validation. +type ConfigValidator interface { + Validate() error +} + type AdapterStats struct { m sync.Mutex lastAck time.Time @@ -127,7 +132,12 @@ func printStruct(prefix string, s interface{}, isTop bool) { } func printUsage() { - logError("Usage: ./adapter adapter_type [config_file.yaml | ...]") + logError("Usage: ./adapter [--validate] [--test-parsing ] adapter_type [config_file.yaml | ...]") + logError("") + logError("Flags:") + logError(" --validate Validate configuration without running the adapter") + logError(" --test-parsing Test parsing with sample data file (requires OID and API key)") + logError("") logError("Available configs:\n") printStruct("", Configuration{}, true) } @@ -140,15 +150,36 @@ func printConfig(method string, c interface{}) { func main() { log("starting") - if len(os.Args) > 1 && strings.HasPrefix(os.Args[1], "-") { - if err := serviceMode(os.Args[0], os.Args[1], os.Args[2:]); err != nil { + // Check for --validate and --test-parsing flags before other processing + validateOnly := false + testParsingFile := "" + args := os.Args[1:] + + for len(args) > 0 { + if args[0] == "--validate" { + validateOnly = true + args = args[1:] + } else if args[0] == "--test-parsing" { + if len(args) < 2 { + logError("--test-parsing requires a sample file path") + os.Exit(1) + } + testParsingFile = args[1] + args = args[2:] + } else { + break + } + } + + if len(args) > 0 && strings.HasPrefix(args[0], "-") { + if err := serviceMode(os.Args[0], args[0], args[1:]); err != nil { logError("service: %v", err) os.Exit(1) } return } - method, configsToRun, err := parseConfigs(os.Args[1:]) + method, configsToRun, err := parseConfigs(args) if err != nil { printUsage() logError("\nerror: %s", err) @@ -159,6 +190,40 @@ func main() { os.Exit(1) return } + + // Handle --validate flag: validate config and exit without running the adapter + if validateOnly { + hasError := false + for i, config := range configsToRun { + log("validating config %d for adapter: %s", i+1, method) + if err := validateConfig(method, config); err != nil { + logError("config %d validation failed: %v", i+1, err) + hasError = true + } else { + log("config %d validation passed", i+1) + } + } + if hasError { + os.Exit(1) + } + log("all configs validated successfully") + return + } + + // Handle --test-parsing flag: test parsing with sample data via API + if testParsingFile != "" { + if len(configsToRun) == 0 { + logError("no configs to test parsing with") + os.Exit(1) + } + config := configsToRun[0] + if err := testParsing(method, config, testParsingFile); err != nil { + logError("parsing test failed: %v", err) + os.Exit(1) + } + return + } + mCurrentlyRunning := sync.Mutex{} clients := []USPClient{} chRunnings := make(chan struct{}) @@ -279,6 +344,333 @@ func main() { log("exited") } +// testParsing tests the parsing configuration with sample data via the LimaCharlie API. +// It reads sample data from the specified file and sends it to the validation API +// to verify that the parsing rules correctly transform the data. +// +// Parameters: +// +// method - The adapter type name (e.g., "syslog", "wel", "s3"). +// configs - The configuration containing parsing/mapping settings. +// sampleFile - Path to a file containing sample data to test. +// +// Returns: +// +// error - Returns nil if parsing succeeds, or an error describing the failure. +func testParsing(method string, configs *Configuration, sampleFile string) error { + // Read sample data from file + sampleData, err := os.ReadFile(sampleFile) + if err != nil { + return fmt.Errorf("failed to read sample file %s: %v", sampleFile, err) + } + + // Get the client options for the adapter to extract OID, API key, platform, and mapping + clientOpts, err := getClientOptions(method, configs) + if err != nil { + return fmt.Errorf("failed to get client options: %v", err) + } + + // Validate we have required credentials + if clientOpts.Identity.Oid == "" { + return errors.New("missing OID in client_options.identity.oid (required for API validation)") + } + if clientOpts.Identity.InstallationKey == "" { + return errors.New("missing API key in client_options.identity.installation_key (required for API validation)") + } + + // Create LimaCharlie client and organization for API authentication + lcClient, err := limacharlie.NewClient(limacharlie.ClientOptions{ + OID: clientOpts.Identity.Oid, + APIKey: clientOpts.Identity.InstallationKey, + }, nil) + if err != nil { + return fmt.Errorf("failed to create LimaCharlie client: %v", err) + } + + org, err := limacharlie.NewOrganization(lcClient) + if err != nil { + return fmt.Errorf("failed to authenticate with LimaCharlie: %v", err) + } + + // Exchange API key for JWT (required for API calls) + if _, err := lcClient.RefreshJWT(time.Hour); err != nil { + return fmt.Errorf("failed to authenticate (check API key): %v", err) + } + + // Build the validation request + req := limacharlie.USPMappingValidationRequest{ + Platform: clientOpts.Platform, + TextInput: string(sampleData), + } + + + // Add mapping if configured (convert to Dict format for API) + if clientOpts.Mapping.ParsingRE != "" || clientOpts.Mapping.EventTypePath != "" || clientOpts.Mapping.Transform != nil { + mappingDict := limacharlie.Dict{} + + if clientOpts.Mapping.ParsingRE != "" { + mappingDict["parsing_re"] = clientOpts.Mapping.ParsingRE + } + if clientOpts.Mapping.EventTypePath != "" { + mappingDict["event_type_path"] = clientOpts.Mapping.EventTypePath + } + if clientOpts.Mapping.EventTimePath != "" { + mappingDict["event_time_path"] = clientOpts.Mapping.EventTimePath + } + if clientOpts.Mapping.SensorHostnamePath != "" { + mappingDict["sensor_hostname_path"] = clientOpts.Mapping.SensorHostnamePath + } + if clientOpts.Mapping.SensorKeyPath != "" { + mappingDict["sensor_key_path"] = clientOpts.Mapping.SensorKeyPath + } + if clientOpts.Mapping.Transform != nil { + mappingDict["transform"] = clientOpts.Mapping.Transform + } + if len(clientOpts.Mapping.Mappings) > 0 { + mappingDict["mappings"] = clientOpts.Mapping.Mappings + } + + req.Mapping = mappingDict + } + + // Add mappings array if configured (for multi-mapping selection) + if len(clientOpts.Mappings) > 0 { + mappingsArray := make([]limacharlie.Dict, 0, len(clientOpts.Mappings)) + for _, m := range clientOpts.Mappings { + md := limacharlie.Dict{} + if m.ParsingRE != "" { + md["parsing_re"] = m.ParsingRE + } + if m.EventTypePath != "" { + md["event_type_path"] = m.EventTypePath + } + if m.EventTimePath != "" { + md["event_time_path"] = m.EventTimePath + } + if m.Transform != nil { + md["transform"] = m.Transform + } + mappingsArray = append(mappingsArray, md) + } + req.Mappings = mappingsArray + } + + log("testing parsing with platform=%s", clientOpts.Platform) + + // Call the validation API via SDK + result, err := org.ValidateUSPMapping(req) + if err != nil { + return fmt.Errorf("API call failed: %v", err) + } + + // Check and display results + return checkParsingResults(result) +} + +// checkParsingResults validates the parsing results and returns an error if validation failed. +// It checks for API errors and empty results (which likely indicate misconfigured parsing rules). +// +// Parameters: +// +// result - The parsing validation result from the API. +// +// Returns: +// +// error - Returns nil if validation passed with at least one event, or an error describing the failure. +func checkParsingResults(result *limacharlie.USPMappingValidationResponse) error { + // Check for errors from the API + if len(result.Errors) > 0 { + log("PARSING FAILED") + log("") + log("Errors:") + for _, e := range result.Errors { + log(" - %s", e) + } + return errors.New("parsing validation failed") + } + + // Check for empty results - this likely indicates a misconfigured parsing rule or platform + if len(result.Results) == 0 { + log("PARSING FAILED") + log("") + log("WARNING: No events were parsed from the sample data.") + log("") + log("This usually indicates one of the following issues:") + log(" - The parsing_re regex does not match the input format") + log(" - The platform type does not match the data format") + log(" - The sample data is empty or contains only whitespace") + log("") + log("Suggestions:") + log(" - Verify your parsing_re regex matches the sample data") + log(" - Check that the platform matches your data format (text, json, cef, etc.)") + log(" - Ensure the sample file contains valid log data") + return errors.New("no events parsed from sample data") + } + + // Display results + log("PARSING SUCCESSFUL") + log("") + log("Parsed %d event(s):", len(result.Results)) + log("") + + for i, event := range result.Results { + log("Event %d:", i+1) + eventJSON, _ := json.MarshalIndent(event, " ", " ") + log(" %s", string(eventJSON)) + log("") + } + + return nil +} + +// getClientOptions extracts the ClientOptions from the config based on adapter type. +// Uses reflection to find the adapter config field by matching the json tag to the method name, +// then extracts the embedded ClientOptions field. +// +// Parameters: +// +// method - The adapter type name (must match the json tag on GeneralConfigs field). +// configs - The configuration struct. +// +// Returns: +// +// *uspclient.ClientOptions - The client options for the adapter. +// error - An error if the adapter type is unknown or doesn't have ClientOptions. +func getClientOptions(method string, configs *Configuration) (*uspclient.ClientOptions, error) { + // Get the embedded GeneralConfigs struct via pointer to make fields addressable + v := reflect.ValueOf(&configs.GeneralConfigs).Elem() + t := v.Type() + + // Iterate over fields to find the one with matching json tag + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + jsonTag := field.Tag.Get("json") + + // Extract the tag name (before any comma options) + tagName := strings.Split(jsonTag, ",")[0] + if tagName != method { + continue + } + + // Found the matching field, now get its ClientOptions + fieldValue := v.Field(i) + + // Look for ClientOptions field in the adapter config struct + clientOptsField := fieldValue.FieldByName("ClientOptions") + if !clientOptsField.IsValid() { + return nil, fmt.Errorf("adapter %s config does not have ClientOptions field", method) + } + + // Return pointer to the ClientOptions + return clientOptsField.Addr().Interface().(*uspclient.ClientOptions), nil + } + + return nil, fmt.Errorf("unknown adapter type: %s", method) +} + +// validateConfig validates the configuration for the specified adapter type. +// It checks that all required fields are present and properly formatted +// without actually starting the adapter. +// +// Parameters: +// +// method - The adapter type name (e.g., "syslog", "wel", "s3"). +// configs - The configuration to validate. +// +// Returns: +// +// error - Returns nil if validation passes, or an error describing the validation failure. +func validateConfig(method string, configs *Configuration) error { + var validator ConfigValidator + + switch method { + case "syslog": + validator = &configs.Syslog + case "pubsub": + validator = &configs.PubSub + case "gcs": + validator = &configs.Gcs + case "s3": + validator = &configs.S3 + case "stdin": + validator = &configs.Stdin + case "1password": + validator = &configs.OnePassword + case "bitwarden": + validator = &configs.Bitwarden + case "itglue": + validator = &configs.ITGlue + case "sophos": + validator = &configs.Sophos + case "okta": + validator = &configs.Okta + case "office365": + validator = &configs.Office365 + case "wiz": + validator = &configs.Wiz + case "wel": + validator = &configs.Wel + case "mac_unified_logging": + validator = &configs.MacUnifiedLogging + case "azure_event_hub": + validator = &configs.AzureEventHub + case "duo": + validator = &configs.Duo + case "cato": + validator = &configs.Cato + case "cylance": + validator = &configs.Cylance + case "entraid": + validator = &configs.EntraID + case "defender": + validator = &configs.Defender + case "slack": + validator = &configs.Slack + case "sqs": + validator = &configs.Sqs + case "sqs-files": + validator = &configs.SqsFiles + case "simulator": + validator = &configs.Simulator + case "file": + validator = &configs.File + case "evtx": + validator = &configs.Evtx + case "k8s_pods": + validator = &configs.K8sPods + case "bigquery": + validator = &configs.BigQuery + case "imap": + validator = &configs.Imap + case "hubspot": + validator = &configs.HubSpot + case "falconcloud": + validator = &configs.FalconCloud + case "mimecast": + validator = &configs.Mimecast + case "ms_graph": + validator = &configs.MsGraph + case "zendesk": + validator = &configs.Zendesk + case "pandadoc": + validator = &configs.PandaDoc + case "proofpoint_tap": + validator = &configs.ProofpointTap + case "box": + validator = &configs.Box + case "sublime": + validator = &configs.Sublime + case "sentinel_one": + validator = &configs.SentinelOne + case "trendmicro": + validator = &configs.TrendMicro + default: + return fmt.Errorf("unknown adapter type: %s", method) + } + + return validator.Validate() +} + func runAdapter(ctx context.Context, method string, configs Configuration, showConfig bool) (USPClient, chan struct{}, error) { var client USPClient var chRunning chan struct{} diff --git a/containers/general/validate_test.go b/containers/general/validate_test.go new file mode 100644 index 0000000..5f969dd --- /dev/null +++ b/containers/general/validate_test.go @@ -0,0 +1,594 @@ +package main + +import ( + "strings" + "testing" + + "github.com/refractionPOINT/go-limacharlie/limacharlie" + "github.com/refractionPOINT/go-uspclient" + usp_bigquery "github.com/refractionPOINT/usp-adapters/bigquery" + usp_file "github.com/refractionPOINT/usp-adapters/file" + usp_k8s_pods "github.com/refractionPOINT/usp-adapters/k8s_pods" + usp_syslog "github.com/refractionPOINT/usp-adapters/syslog" + usp_wel "github.com/refractionPOINT/usp-adapters/wel" +) + +func TestValidateConfig(t *testing.T) { + t.Run("ValidSyslogConfig", func(t *testing.T) { + config := &Configuration{} + config.Syslog = usp_syslog.SyslogConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "text", + }, + Port: 514, + } + + err := validateConfig("syslog", config) + if err != nil { + t.Errorf("expected valid syslog config to pass validation, got error: %v", err) + } + }) + + t.Run("InvalidSyslogConfigMissingIdentity", func(t *testing.T) { + config := &Configuration{} + config.Syslog = usp_syslog.SyslogConfig{ + ClientOptions: uspclient.ClientOptions{}, + Port: 514, + } + + err := validateConfig("syslog", config) + if err == nil { + t.Error("expected syslog config with missing identity to fail validation") + } + }) + + t.Run("ValidWELConfig", func(t *testing.T) { + config := &Configuration{} + config.Wel = usp_wel.WELConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "json", + }, + EvtSources: "Security,System", + } + + err := validateConfig("wel", config) + if err != nil { + t.Errorf("expected valid wel config to pass validation, got error: %v", err) + } + }) + + t.Run("InvalidWELConfigMissingIdentity", func(t *testing.T) { + config := &Configuration{} + config.Wel = usp_wel.WELConfig{ + ClientOptions: uspclient.ClientOptions{}, + EvtSources: "Security", + } + + err := validateConfig("wel", config) + if err == nil { + t.Error("expected wel config with missing identity to fail validation") + } + }) + + t.Run("ValidFileConfig", func(t *testing.T) { + config := &Configuration{} + config.File = usp_file.FileConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "text", + }, + FilePath: "/var/log/test.log", + } + + err := validateConfig("file", config) + if err != nil { + t.Errorf("expected valid file config to pass validation, got error: %v", err) + } + }) + + t.Run("InvalidFileConfigMissingFilePath", func(t *testing.T) { + config := &Configuration{} + config.File = usp_file.FileConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "text", + }, + // FilePath is missing + } + + err := validateConfig("file", config) + if err == nil { + t.Error("expected file config with missing file_path to fail validation") + } + }) + + t.Run("InvalidFileConfigMissingIdentity", func(t *testing.T) { + config := &Configuration{} + config.File = usp_file.FileConfig{ + ClientOptions: uspclient.ClientOptions{}, + FilePath: "/var/log/test.log", + } + + err := validateConfig("file", config) + if err == nil { + t.Error("expected file config with missing identity to fail validation") + } + }) + + t.Run("ValidK8sPodsConfig", func(t *testing.T) { + config := &Configuration{} + config.K8sPods = usp_k8s_pods.K8sPodsConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "json", + }, + Root: "/var/log/pods", + } + + err := validateConfig("k8s_pods", config) + if err != nil { + t.Errorf("expected valid k8s_pods config to pass validation, got error: %v", err) + } + }) + + t.Run("InvalidK8sPodsConfigMissingRoot", func(t *testing.T) { + config := &Configuration{} + config.K8sPods = usp_k8s_pods.K8sPodsConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "json", + }, + // Root is missing + } + + err := validateConfig("k8s_pods", config) + if err == nil { + t.Error("expected k8s_pods config with missing root to fail validation") + } + }) + + t.Run("InvalidK8sPodsConfigMissingIdentity", func(t *testing.T) { + config := &Configuration{} + config.K8sPods = usp_k8s_pods.K8sPodsConfig{ + ClientOptions: uspclient.ClientOptions{}, + Root: "/var/log/pods", + } + + err := validateConfig("k8s_pods", config) + if err == nil { + t.Error("expected k8s_pods config with missing identity to fail validation") + } + }) + + t.Run("ValidBigQueryConfig", func(t *testing.T) { + config := &Configuration{} + config.BigQuery = usp_bigquery.BigQueryConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "json", + }, + ProjectId: "my-project", + BigQueryProject: "my-project", + DatasetName: "my-dataset", + TableName: "my-table", + SqlQuery: "SELECT * FROM my-table", + } + + err := validateConfig("bigquery", config) + if err != nil { + t.Errorf("expected valid bigquery config to pass validation, got error: %v", err) + } + }) + + t.Run("InvalidBigQueryConfigMissingProjectId", func(t *testing.T) { + config := &Configuration{} + config.BigQuery = usp_bigquery.BigQueryConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "json", + }, + // ProjectId is missing + BigQueryProject: "my-project", + DatasetName: "my-dataset", + TableName: "my-table", + SqlQuery: "SELECT * FROM my-table", + } + + err := validateConfig("bigquery", config) + if err == nil { + t.Error("expected bigquery config with missing project_id to fail validation") + } + }) + + t.Run("InvalidBigQueryConfigMissingSqlQuery", func(t *testing.T) { + config := &Configuration{} + config.BigQuery = usp_bigquery.BigQueryConfig{ + ClientOptions: uspclient.ClientOptions{ + Identity: uspclient.Identity{ + Oid: "test-oid", + InstallationKey: "test-key", + }, + Platform: "json", + }, + ProjectId: "my-project", + BigQueryProject: "my-project", + DatasetName: "my-dataset", + TableName: "my-table", + // SqlQuery is missing + } + + err := validateConfig("bigquery", config) + if err == nil { + t.Error("expected bigquery config with missing sql_query to fail validation") + } + }) + + t.Run("UnknownAdapterType", func(t *testing.T) { + config := &Configuration{} + err := validateConfig("unknown_adapter", config) + if err == nil { + t.Error("expected unknown adapter type to fail validation") + } + if err.Error() != "unknown adapter type: unknown_adapter" { + t.Errorf("expected specific error message, got: %v", err) + } + }) + + t.Run("AllKnownAdapterTypesSupported", func(t *testing.T) { + // List of all adapter types that should be supported + adapterTypes := []string{ + "syslog", "pubsub", "gcs", "s3", "stdin", "1password", "bitwarden", + "itglue", "sophos", "okta", "office365", "wiz", "wel", "mac_unified_logging", + "azure_event_hub", "duo", "cato", "cylance", "entraid", "defender", + "slack", "sqs", "sqs-files", "simulator", "file", "evtx", "k8s_pods", + "bigquery", "imap", "hubspot", "falconcloud", "mimecast", "ms_graph", + "zendesk", "pandadoc", "proofpoint_tap", "box", "sublime", + "sentinel_one", "trendmicro", + } + + config := &Configuration{} + for _, adapterType := range adapterTypes { + err := validateConfig(adapterType, config) + // We expect errors due to missing identity, but NOT "unknown adapter type" + if err != nil && err.Error() == "unknown adapter type: "+adapterType { + t.Errorf("adapter type %q should be supported but got unknown adapter error", adapterType) + } + } + }) +} + +// TestGetClientOptions tests the reflection-based getClientOptions function +// that extracts ClientOptions from adapter configs based on json tags. +func TestGetClientOptions(t *testing.T) { + t.Run("Syslog", func(t *testing.T) { + config := &Configuration{} + config.Syslog.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "test-oid-syslog"}, + } + + opts, err := getClientOptions("syslog", config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts == nil { + t.Fatal("expected non-nil ClientOptions") + } + if opts.Identity.Oid != "test-oid-syslog" { + t.Errorf("expected Oid 'test-oid-syslog', got %q", opts.Identity.Oid) + } + }) + + t.Run("WEL", func(t *testing.T) { + config := &Configuration{} + config.Wel.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "test-oid-wel"}, + } + + opts, err := getClientOptions("wel", config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Identity.Oid != "test-oid-wel" { + t.Errorf("expected Oid 'test-oid-wel', got %q", opts.Identity.Oid) + } + }) + + t.Run("S3", func(t *testing.T) { + config := &Configuration{} + config.S3.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "test-oid-s3"}, + } + + opts, err := getClientOptions("s3", config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Identity.Oid != "test-oid-s3" { + t.Errorf("expected Oid 'test-oid-s3', got %q", opts.Identity.Oid) + } + }) + + t.Run("SqsFiles", func(t *testing.T) { + // Test adapter with hyphen in name (sqs-files) + config := &Configuration{} + config.SqsFiles.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "test-oid-sqs-files"}, + } + + opts, err := getClientOptions("sqs-files", config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Identity.Oid != "test-oid-sqs-files" { + t.Errorf("expected Oid 'test-oid-sqs-files', got %q", opts.Identity.Oid) + } + }) + + t.Run("OnePassword", func(t *testing.T) { + // Test adapter with numeric prefix (1password) + config := &Configuration{} + config.OnePassword.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "test-oid-1password"}, + } + + opts, err := getClientOptions("1password", config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Identity.Oid != "test-oid-1password" { + t.Errorf("expected Oid 'test-oid-1password', got %q", opts.Identity.Oid) + } + }) + + t.Run("UnknownAdapter", func(t *testing.T) { + config := &Configuration{} + _, err := getClientOptions("unknown_adapter", config) + if err == nil { + t.Fatal("expected error for unknown adapter") + } + if err.Error() != "unknown adapter type: unknown_adapter" { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("ReturnsPointerToActualField", func(t *testing.T) { + // Verify that modifying the returned pointer modifies the original config + config := &Configuration{} + config.Syslog.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "original"}, + } + + opts, err := getClientOptions("syslog", config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Modify via the returned pointer + opts.Identity.Oid = "modified" + + // Verify the original config was modified + if config.Syslog.ClientOptions.Identity.Oid != "modified" { + t.Error("expected modification via returned pointer to affect original config") + } + }) + + t.Run("AllKnownAdapterTypes", func(t *testing.T) { + // Comprehensive test that all adapter types work with reflection + adapterTypes := []string{ + "syslog", "pubsub", "gcs", "s3", "stdin", "1password", "bitwarden", + "itglue", "sophos", "okta", "office365", "wiz", "wel", "mac_unified_logging", + "azure_event_hub", "duo", "cato", "cylance", "entraid", "defender", + "slack", "sqs", "sqs-files", "simulator", "file", "evtx", "k8s_pods", + "bigquery", "imap", "hubspot", "falconcloud", "mimecast", "ms_graph", + "zendesk", "pandadoc", "proofpoint_tap", "box", "sublime", + "sentinel_one", "trendmicro", + } + + config := &Configuration{} + for _, adapterType := range adapterTypes { + opts, err := getClientOptions(adapterType, config) + if err != nil { + t.Errorf("adapter %q: unexpected error: %v", adapterType, err) + continue + } + if opts == nil { + t.Errorf("adapter %q: expected non-nil ClientOptions", adapterType) + } + } + }) + + t.Run("MatchesJsonTagNotFieldName", func(t *testing.T) { + // Verify reflection uses json tag, not Go field name + // Field name is "SentinelOne" but json tag is "sentinel_one" + config := &Configuration{} + config.SentinelOne.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "test-sentinel"}, + } + + // Should work with json tag name + opts, err := getClientOptions("sentinel_one", config) + if err != nil { + t.Fatalf("expected json tag 'sentinel_one' to work: %v", err) + } + if opts.Identity.Oid != "test-sentinel" { + t.Errorf("expected Oid 'test-sentinel', got %q", opts.Identity.Oid) + } + + // Should NOT work with Go field name + _, err = getClientOptions("SentinelOne", config) + if err == nil { + t.Error("expected Go field name 'SentinelOne' to fail (should use json tag)") + } + }) + + t.Run("MultipleConfigsIndependent", func(t *testing.T) { + // Verify getting options for one adapter doesn't affect another + config := &Configuration{} + config.Syslog.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "syslog-oid"}, + } + config.S3.ClientOptions = uspclient.ClientOptions{ + Identity: uspclient.Identity{Oid: "s3-oid"}, + } + + syslogOpts, _ := getClientOptions("syslog", config) + s3Opts, _ := getClientOptions("s3", config) + + if syslogOpts.Identity.Oid != "syslog-oid" { + t.Errorf("syslog Oid changed unexpectedly: %q", syslogOpts.Identity.Oid) + } + if s3Opts.Identity.Oid != "s3-oid" { + t.Errorf("s3 Oid changed unexpectedly: %q", s3Opts.Identity.Oid) + } + }) +} + +func TestConfigValidatorInterface(t *testing.T) { + t.Run("SyslogConfigImplementsInterface", func(t *testing.T) { + var _ ConfigValidator = &usp_syslog.SyslogConfig{} + }) + + t.Run("WELConfigImplementsInterface", func(t *testing.T) { + var _ ConfigValidator = &usp_wel.WELConfig{} + }) + + t.Run("FileConfigImplementsInterface", func(t *testing.T) { + var _ ConfigValidator = &usp_file.FileConfig{} + }) + + t.Run("K8sPodsConfigImplementsInterface", func(t *testing.T) { + var _ ConfigValidator = &usp_k8s_pods.K8sPodsConfig{} + }) + + t.Run("BigQueryConfigImplementsInterface", func(t *testing.T) { + var _ ConfigValidator = &usp_bigquery.BigQueryConfig{} + }) +} + +// TestCheckParsingResults tests the checkParsingResults function which validates +// the response from the LimaCharlie USP parsing API. +func TestCheckParsingResults(t *testing.T) { + t.Run("SuccessfulParsingWithResults", func(t *testing.T) { + // Successful parsing returns at least one event + result := &limacharlie.USPMappingValidationResponse{ + Results: []limacharlie.Dict{ + {"event_type": "INFO", "message": "test event 1"}, + {"event_type": "WARN", "message": "test event 2"}, + }, + Errors: []string{}, + } + + err := checkParsingResults(result) + if err != nil { + t.Errorf("expected successful parsing to return nil, got error: %v", err) + } + }) + + t.Run("ParsingWithAPIErrors", func(t *testing.T) { + // API returns errors indicating parsing failure + result := &limacharlie.USPMappingValidationResponse{ + Results: []limacharlie.Dict{}, + Errors: []string{"regex pattern did not match", "invalid mapping configuration"}, + } + + err := checkParsingResults(result) + if err == nil { + t.Error("expected parsing with API errors to return error") + } + if !strings.Contains(err.Error(), "parsing validation failed") { + t.Errorf("expected error message to contain 'parsing validation failed', got: %v", err) + } + }) + + t.Run("EmptyResultsNoErrors", func(t *testing.T) { + // Empty results with no API errors - indicates regex doesn't match + // or wrong platform type + result := &limacharlie.USPMappingValidationResponse{ + Results: []limacharlie.Dict{}, + Errors: []string{}, + } + + err := checkParsingResults(result) + if err == nil { + t.Error("expected empty results to return error") + } + if !strings.Contains(err.Error(), "no events parsed") { + t.Errorf("expected error message to contain 'no events parsed', got: %v", err) + } + }) + + t.Run("NilResults", func(t *testing.T) { + // Nil results should be treated as empty results + result := &limacharlie.USPMappingValidationResponse{ + Results: nil, + Errors: nil, + } + + err := checkParsingResults(result) + if err == nil { + t.Error("expected nil results to return error") + } + if !strings.Contains(err.Error(), "no events parsed") { + t.Errorf("expected error message to contain 'no events parsed', got: %v", err) + } + }) + + t.Run("PartialParsingWithErrors", func(t *testing.T) { + // Some results but also some errors - errors take precedence + result := &limacharlie.USPMappingValidationResponse{ + Results: []limacharlie.Dict{ + {"event_type": "INFO", "message": "partial event"}, + }, + Errors: []string{"some lines failed to parse"}, + } + + err := checkParsingResults(result) + if err == nil { + t.Error("expected partial parsing with errors to return error") + } + if !strings.Contains(err.Error(), "parsing validation failed") { + t.Errorf("expected error message to contain 'parsing validation failed', got: %v", err) + } + }) + + t.Run("SingleEventSuccess", func(t *testing.T) { + // Single event is still a success + result := &limacharlie.USPMappingValidationResponse{ + Results: []limacharlie.Dict{ + {"event_type": "INFO"}, + }, + Errors: []string{}, + } + + err := checkParsingResults(result) + if err != nil { + t.Errorf("expected single event parsing to return nil, got error: %v", err) + } + }) +} diff --git a/file/client.go b/file/client.go index 258c264..e2ba0bd 100644 --- a/file/client.go +++ b/file/client.go @@ -5,7 +5,6 @@ package usp_file import ( "context" - "errors" "fmt" "io" "os" @@ -68,16 +67,6 @@ type FileAdapter struct { lineCb func(line string) // callback for each line for testing } -func (c *FileConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.FilePath == "" { - return errors.New("file_path missing") - } - return nil -} - func NewFileAdapter(ctx context.Context, conf FileConfig) (*FileAdapter, chan struct{}, error) { a := &FileAdapter{ conf: conf, diff --git a/file/conf.go b/file/conf.go index 6450e23..065b689 100644 --- a/file/conf.go +++ b/file/conf.go @@ -1,6 +1,9 @@ package usp_file import ( + "errors" + "fmt" + "github.com/refractionPOINT/go-uspclient" ) @@ -16,3 +19,22 @@ type FileConfig struct { Poll bool `json:"poll" yaml:"poll"` MultiLineJSON bool `json:"multi_line_json" yaml:"multi_line_json"` } + +// Validate validates the File adapter configuration. +// +// Parameters: +// +// None +// +// Returns: +// +// error - Returns nil if validation passes, or an error describing the validation failure. +func (c *FileConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.FilePath == "" { + return errors.New("file_path missing") + } + return nil +} diff --git a/k8s_pods/client.go b/k8s_pods/client.go index 8497bed..7fdf938 100644 --- a/k8s_pods/client.go +++ b/k8s_pods/client.go @@ -5,7 +5,6 @@ package usp_k8s_pods import ( "context" - "errors" "fmt" "regexp" "sync" @@ -36,16 +35,6 @@ type runtimeOptions struct { excludePods *regexp.Regexp } -func (c *K8sPodsConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Root == "" { - return errors.New("file_path missing") - } - return nil -} - func NewK8sPodsAdapter(ctx context.Context, conf K8sPodsConfig) (*K8sPodsAdapter, chan struct{}, error) { a := &K8sPodsAdapter{ conf: conf, diff --git a/k8s_pods/conf.go b/k8s_pods/conf.go index 8cdd307..e8a962a 100644 --- a/k8s_pods/conf.go +++ b/k8s_pods/conf.go @@ -1,6 +1,9 @@ package usp_k8s_pods import ( + "errors" + "fmt" + "github.com/refractionPOINT/go-uspclient" ) @@ -11,3 +14,22 @@ type K8sPodsConfig struct { IncludePodsRE string `json:"include_pods_re" yaml:"include_pods_re"` ExcludePodsRE string `json:"exclude_pods_re" yaml:"exclude_pods_re"` } + +// Validate validates the K8s Pods adapter configuration. +// +// Parameters: +// +// None +// +// Returns: +// +// error - Returns nil if validation passes, or an error describing the validation failure. +func (c *K8sPodsConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Root == "" { + return errors.New("root missing") + } + return nil +}