diff --git a/bigquery/client.go b/bigquery/client.go
index c6aa534..2a6ae12 100644
--- a/bigquery/client.go
+++ b/bigquery/client.go
@@ -30,26 +30,6 @@ type BigQueryAdapter struct {
cancel context.CancelFunc
}
-func (bq *BigQueryConfig) Validate() error {
- if bq.ProjectId == "" {
- return errors.New("missing project_id")
- }
- // this will usually be th same as projectID but could be different if using outside project dataset such as a public data set
- if bq.BigQueryProject == "" {
- return errors.New("missing bigquery project name")
- }
- if bq.DatasetName == "" {
- return errors.New("missing dataset_name")
- }
- if bq.TableName == "" {
- return errors.New("missing table_name")
- }
- if bq.SqlQuery == "" {
- return errors.New("missing sql query")
- }
- return nil
-}
-
func NewBigQueryAdapter(ctx context.Context, conf BigQueryConfig) (*BigQueryAdapter, chan struct{}, error) {
// Create bq cancellable context
ctx, cancel := context.WithCancel(context.Background())
diff --git a/bigquery/conf.go b/bigquery/conf.go
index 56037e9..07c225c 100644
--- a/bigquery/conf.go
+++ b/bigquery/conf.go
@@ -1,6 +1,8 @@
package usp_bigquery
import (
+ "errors"
+
"github.com/refractionPOINT/go-uspclient"
)
@@ -15,3 +17,33 @@ type BigQueryConfig struct {
QueryInterval string `json:"query_interval" yaml:"query_interval"`
IsOneTimeLoad bool `json:"is_one_time_load" yaml:"is_one_time_load"`
}
+
+// Validate validates the BigQuery adapter configuration.
+//
+// Parameters:
+//
+// None
+//
+// Returns:
+//
+// error - Returns nil if validation passes, or an error describing the validation failure.
+func (c *BigQueryConfig) Validate() error {
+ if c.ProjectId == "" {
+ return errors.New("missing project_id")
+ }
+ // BigQueryProject will usually be the same as ProjectId but could be different
+ // if using outside project dataset such as a public data set.
+ if c.BigQueryProject == "" {
+ return errors.New("missing bigquery_project")
+ }
+ if c.DatasetName == "" {
+ return errors.New("missing dataset_name")
+ }
+ if c.TableName == "" {
+ return errors.New("missing table_name")
+ }
+ if c.SqlQuery == "" {
+ return errors.New("missing sql_query")
+ }
+ return nil
+}
diff --git a/containers/general/tool.go b/containers/general/tool.go
index 4c7af0b..1c67b15 100755
--- a/containers/general/tool.go
+++ b/containers/general/tool.go
@@ -69,6 +69,11 @@ type USPClient interface {
Close() error
}
+// ConfigValidator is implemented by adapter configs that support validation.
+type ConfigValidator interface {
+ Validate() error
+}
+
type AdapterStats struct {
m sync.Mutex
lastAck time.Time
@@ -127,7 +132,12 @@ func printStruct(prefix string, s interface{}, isTop bool) {
}
func printUsage() {
- logError("Usage: ./adapter adapter_type [config_file.yaml | ...]")
+ logError("Usage: ./adapter [--validate] [--test-parsing ] adapter_type [config_file.yaml | ...]")
+ logError("")
+ logError("Flags:")
+ logError(" --validate Validate configuration without running the adapter")
+ logError(" --test-parsing Test parsing with sample data file (requires OID and API key)")
+ logError("")
logError("Available configs:\n")
printStruct("", Configuration{}, true)
}
@@ -140,15 +150,36 @@ func printConfig(method string, c interface{}) {
func main() {
log("starting")
- if len(os.Args) > 1 && strings.HasPrefix(os.Args[1], "-") {
- if err := serviceMode(os.Args[0], os.Args[1], os.Args[2:]); err != nil {
+ // Check for --validate and --test-parsing flags before other processing
+ validateOnly := false
+ testParsingFile := ""
+ args := os.Args[1:]
+
+ for len(args) > 0 {
+ if args[0] == "--validate" {
+ validateOnly = true
+ args = args[1:]
+ } else if args[0] == "--test-parsing" {
+ if len(args) < 2 {
+ logError("--test-parsing requires a sample file path")
+ os.Exit(1)
+ }
+ testParsingFile = args[1]
+ args = args[2:]
+ } else {
+ break
+ }
+ }
+
+ if len(args) > 0 && strings.HasPrefix(args[0], "-") {
+ if err := serviceMode(os.Args[0], args[0], args[1:]); err != nil {
logError("service: %v", err)
os.Exit(1)
}
return
}
- method, configsToRun, err := parseConfigs(os.Args[1:])
+ method, configsToRun, err := parseConfigs(args)
if err != nil {
printUsage()
logError("\nerror: %s", err)
@@ -159,6 +190,40 @@ func main() {
os.Exit(1)
return
}
+
+ // Handle --validate flag: validate config and exit without running the adapter
+ if validateOnly {
+ hasError := false
+ for i, config := range configsToRun {
+ log("validating config %d for adapter: %s", i+1, method)
+ if err := validateConfig(method, config); err != nil {
+ logError("config %d validation failed: %v", i+1, err)
+ hasError = true
+ } else {
+ log("config %d validation passed", i+1)
+ }
+ }
+ if hasError {
+ os.Exit(1)
+ }
+ log("all configs validated successfully")
+ return
+ }
+
+ // Handle --test-parsing flag: test parsing with sample data via API
+ if testParsingFile != "" {
+ if len(configsToRun) == 0 {
+ logError("no configs to test parsing with")
+ os.Exit(1)
+ }
+ config := configsToRun[0]
+ if err := testParsing(method, config, testParsingFile); err != nil {
+ logError("parsing test failed: %v", err)
+ os.Exit(1)
+ }
+ return
+ }
+
mCurrentlyRunning := sync.Mutex{}
clients := []USPClient{}
chRunnings := make(chan struct{})
@@ -279,6 +344,333 @@ func main() {
log("exited")
}
+// testParsing tests the parsing configuration with sample data via the LimaCharlie API.
+// It reads sample data from the specified file and sends it to the validation API
+// to verify that the parsing rules correctly transform the data.
+//
+// Parameters:
+//
+// method - The adapter type name (e.g., "syslog", "wel", "s3").
+// configs - The configuration containing parsing/mapping settings.
+// sampleFile - Path to a file containing sample data to test.
+//
+// Returns:
+//
+// error - Returns nil if parsing succeeds, or an error describing the failure.
+func testParsing(method string, configs *Configuration, sampleFile string) error {
+ // Read sample data from file
+ sampleData, err := os.ReadFile(sampleFile)
+ if err != nil {
+ return fmt.Errorf("failed to read sample file %s: %v", sampleFile, err)
+ }
+
+ // Get the client options for the adapter to extract OID, API key, platform, and mapping
+ clientOpts, err := getClientOptions(method, configs)
+ if err != nil {
+ return fmt.Errorf("failed to get client options: %v", err)
+ }
+
+ // Validate we have required credentials
+ if clientOpts.Identity.Oid == "" {
+ return errors.New("missing OID in client_options.identity.oid (required for API validation)")
+ }
+ if clientOpts.Identity.InstallationKey == "" {
+ return errors.New("missing API key in client_options.identity.installation_key (required for API validation)")
+ }
+
+ // Create LimaCharlie client and organization for API authentication
+ lcClient, err := limacharlie.NewClient(limacharlie.ClientOptions{
+ OID: clientOpts.Identity.Oid,
+ APIKey: clientOpts.Identity.InstallationKey,
+ }, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create LimaCharlie client: %v", err)
+ }
+
+ org, err := limacharlie.NewOrganization(lcClient)
+ if err != nil {
+ return fmt.Errorf("failed to authenticate with LimaCharlie: %v", err)
+ }
+
+ // Exchange API key for JWT (required for API calls)
+ if _, err := lcClient.RefreshJWT(time.Hour); err != nil {
+ return fmt.Errorf("failed to authenticate (check API key): %v", err)
+ }
+
+ // Build the validation request
+ req := limacharlie.USPMappingValidationRequest{
+ Platform: clientOpts.Platform,
+ TextInput: string(sampleData),
+ }
+
+
+ // Add mapping if configured (convert to Dict format for API)
+ if clientOpts.Mapping.ParsingRE != "" || clientOpts.Mapping.EventTypePath != "" || clientOpts.Mapping.Transform != nil {
+ mappingDict := limacharlie.Dict{}
+
+ if clientOpts.Mapping.ParsingRE != "" {
+ mappingDict["parsing_re"] = clientOpts.Mapping.ParsingRE
+ }
+ if clientOpts.Mapping.EventTypePath != "" {
+ mappingDict["event_type_path"] = clientOpts.Mapping.EventTypePath
+ }
+ if clientOpts.Mapping.EventTimePath != "" {
+ mappingDict["event_time_path"] = clientOpts.Mapping.EventTimePath
+ }
+ if clientOpts.Mapping.SensorHostnamePath != "" {
+ mappingDict["sensor_hostname_path"] = clientOpts.Mapping.SensorHostnamePath
+ }
+ if clientOpts.Mapping.SensorKeyPath != "" {
+ mappingDict["sensor_key_path"] = clientOpts.Mapping.SensorKeyPath
+ }
+ if clientOpts.Mapping.Transform != nil {
+ mappingDict["transform"] = clientOpts.Mapping.Transform
+ }
+ if len(clientOpts.Mapping.Mappings) > 0 {
+ mappingDict["mappings"] = clientOpts.Mapping.Mappings
+ }
+
+ req.Mapping = mappingDict
+ }
+
+ // Add mappings array if configured (for multi-mapping selection)
+ if len(clientOpts.Mappings) > 0 {
+ mappingsArray := make([]limacharlie.Dict, 0, len(clientOpts.Mappings))
+ for _, m := range clientOpts.Mappings {
+ md := limacharlie.Dict{}
+ if m.ParsingRE != "" {
+ md["parsing_re"] = m.ParsingRE
+ }
+ if m.EventTypePath != "" {
+ md["event_type_path"] = m.EventTypePath
+ }
+ if m.EventTimePath != "" {
+ md["event_time_path"] = m.EventTimePath
+ }
+ if m.Transform != nil {
+ md["transform"] = m.Transform
+ }
+ mappingsArray = append(mappingsArray, md)
+ }
+ req.Mappings = mappingsArray
+ }
+
+ log("testing parsing with platform=%s", clientOpts.Platform)
+
+ // Call the validation API via SDK
+ result, err := org.ValidateUSPMapping(req)
+ if err != nil {
+ return fmt.Errorf("API call failed: %v", err)
+ }
+
+ // Check and display results
+ return checkParsingResults(result)
+}
+
+// checkParsingResults validates the parsing results and returns an error if validation failed.
+// It checks for API errors and empty results (which likely indicate misconfigured parsing rules).
+//
+// Parameters:
+//
+// result - The parsing validation result from the API.
+//
+// Returns:
+//
+// error - Returns nil if validation passed with at least one event, or an error describing the failure.
+func checkParsingResults(result *limacharlie.USPMappingValidationResponse) error {
+ // Check for errors from the API
+ if len(result.Errors) > 0 {
+ log("PARSING FAILED")
+ log("")
+ log("Errors:")
+ for _, e := range result.Errors {
+ log(" - %s", e)
+ }
+ return errors.New("parsing validation failed")
+ }
+
+ // Check for empty results - this likely indicates a misconfigured parsing rule or platform
+ if len(result.Results) == 0 {
+ log("PARSING FAILED")
+ log("")
+ log("WARNING: No events were parsed from the sample data.")
+ log("")
+ log("This usually indicates one of the following issues:")
+ log(" - The parsing_re regex does not match the input format")
+ log(" - The platform type does not match the data format")
+ log(" - The sample data is empty or contains only whitespace")
+ log("")
+ log("Suggestions:")
+ log(" - Verify your parsing_re regex matches the sample data")
+ log(" - Check that the platform matches your data format (text, json, cef, etc.)")
+ log(" - Ensure the sample file contains valid log data")
+ return errors.New("no events parsed from sample data")
+ }
+
+ // Display results
+ log("PARSING SUCCESSFUL")
+ log("")
+ log("Parsed %d event(s):", len(result.Results))
+ log("")
+
+ for i, event := range result.Results {
+ log("Event %d:", i+1)
+ eventJSON, _ := json.MarshalIndent(event, " ", " ")
+ log(" %s", string(eventJSON))
+ log("")
+ }
+
+ return nil
+}
+
+// getClientOptions extracts the ClientOptions from the config based on adapter type.
+// Uses reflection to find the adapter config field by matching the json tag to the method name,
+// then extracts the embedded ClientOptions field.
+//
+// Parameters:
+//
+// method - The adapter type name (must match the json tag on GeneralConfigs field).
+// configs - The configuration struct.
+//
+// Returns:
+//
+// *uspclient.ClientOptions - The client options for the adapter.
+// error - An error if the adapter type is unknown or doesn't have ClientOptions.
+func getClientOptions(method string, configs *Configuration) (*uspclient.ClientOptions, error) {
+ // Get the embedded GeneralConfigs struct via pointer to make fields addressable
+ v := reflect.ValueOf(&configs.GeneralConfigs).Elem()
+ t := v.Type()
+
+ // Iterate over fields to find the one with matching json tag
+ for i := 0; i < t.NumField(); i++ {
+ field := t.Field(i)
+ jsonTag := field.Tag.Get("json")
+
+ // Extract the tag name (before any comma options)
+ tagName := strings.Split(jsonTag, ",")[0]
+ if tagName != method {
+ continue
+ }
+
+ // Found the matching field, now get its ClientOptions
+ fieldValue := v.Field(i)
+
+ // Look for ClientOptions field in the adapter config struct
+ clientOptsField := fieldValue.FieldByName("ClientOptions")
+ if !clientOptsField.IsValid() {
+ return nil, fmt.Errorf("adapter %s config does not have ClientOptions field", method)
+ }
+
+ // Return pointer to the ClientOptions
+ return clientOptsField.Addr().Interface().(*uspclient.ClientOptions), nil
+ }
+
+ return nil, fmt.Errorf("unknown adapter type: %s", method)
+}
+
+// validateConfig validates the configuration for the specified adapter type.
+// It checks that all required fields are present and properly formatted
+// without actually starting the adapter.
+//
+// Parameters:
+//
+// method - The adapter type name (e.g., "syslog", "wel", "s3").
+// configs - The configuration to validate.
+//
+// Returns:
+//
+// error - Returns nil if validation passes, or an error describing the validation failure.
+func validateConfig(method string, configs *Configuration) error {
+ var validator ConfigValidator
+
+ switch method {
+ case "syslog":
+ validator = &configs.Syslog
+ case "pubsub":
+ validator = &configs.PubSub
+ case "gcs":
+ validator = &configs.Gcs
+ case "s3":
+ validator = &configs.S3
+ case "stdin":
+ validator = &configs.Stdin
+ case "1password":
+ validator = &configs.OnePassword
+ case "bitwarden":
+ validator = &configs.Bitwarden
+ case "itglue":
+ validator = &configs.ITGlue
+ case "sophos":
+ validator = &configs.Sophos
+ case "okta":
+ validator = &configs.Okta
+ case "office365":
+ validator = &configs.Office365
+ case "wiz":
+ validator = &configs.Wiz
+ case "wel":
+ validator = &configs.Wel
+ case "mac_unified_logging":
+ validator = &configs.MacUnifiedLogging
+ case "azure_event_hub":
+ validator = &configs.AzureEventHub
+ case "duo":
+ validator = &configs.Duo
+ case "cato":
+ validator = &configs.Cato
+ case "cylance":
+ validator = &configs.Cylance
+ case "entraid":
+ validator = &configs.EntraID
+ case "defender":
+ validator = &configs.Defender
+ case "slack":
+ validator = &configs.Slack
+ case "sqs":
+ validator = &configs.Sqs
+ case "sqs-files":
+ validator = &configs.SqsFiles
+ case "simulator":
+ validator = &configs.Simulator
+ case "file":
+ validator = &configs.File
+ case "evtx":
+ validator = &configs.Evtx
+ case "k8s_pods":
+ validator = &configs.K8sPods
+ case "bigquery":
+ validator = &configs.BigQuery
+ case "imap":
+ validator = &configs.Imap
+ case "hubspot":
+ validator = &configs.HubSpot
+ case "falconcloud":
+ validator = &configs.FalconCloud
+ case "mimecast":
+ validator = &configs.Mimecast
+ case "ms_graph":
+ validator = &configs.MsGraph
+ case "zendesk":
+ validator = &configs.Zendesk
+ case "pandadoc":
+ validator = &configs.PandaDoc
+ case "proofpoint_tap":
+ validator = &configs.ProofpointTap
+ case "box":
+ validator = &configs.Box
+ case "sublime":
+ validator = &configs.Sublime
+ case "sentinel_one":
+ validator = &configs.SentinelOne
+ case "trendmicro":
+ validator = &configs.TrendMicro
+ default:
+ return fmt.Errorf("unknown adapter type: %s", method)
+ }
+
+ return validator.Validate()
+}
+
func runAdapter(ctx context.Context, method string, configs Configuration, showConfig bool) (USPClient, chan struct{}, error) {
var client USPClient
var chRunning chan struct{}
diff --git a/containers/general/validate_test.go b/containers/general/validate_test.go
new file mode 100644
index 0000000..5f969dd
--- /dev/null
+++ b/containers/general/validate_test.go
@@ -0,0 +1,594 @@
+package main
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/refractionPOINT/go-limacharlie/limacharlie"
+ "github.com/refractionPOINT/go-uspclient"
+ usp_bigquery "github.com/refractionPOINT/usp-adapters/bigquery"
+ usp_file "github.com/refractionPOINT/usp-adapters/file"
+ usp_k8s_pods "github.com/refractionPOINT/usp-adapters/k8s_pods"
+ usp_syslog "github.com/refractionPOINT/usp-adapters/syslog"
+ usp_wel "github.com/refractionPOINT/usp-adapters/wel"
+)
+
+func TestValidateConfig(t *testing.T) {
+ t.Run("ValidSyslogConfig", func(t *testing.T) {
+ config := &Configuration{}
+ config.Syslog = usp_syslog.SyslogConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "text",
+ },
+ Port: 514,
+ }
+
+ err := validateConfig("syslog", config)
+ if err != nil {
+ t.Errorf("expected valid syslog config to pass validation, got error: %v", err)
+ }
+ })
+
+ t.Run("InvalidSyslogConfigMissingIdentity", func(t *testing.T) {
+ config := &Configuration{}
+ config.Syslog = usp_syslog.SyslogConfig{
+ ClientOptions: uspclient.ClientOptions{},
+ Port: 514,
+ }
+
+ err := validateConfig("syslog", config)
+ if err == nil {
+ t.Error("expected syslog config with missing identity to fail validation")
+ }
+ })
+
+ t.Run("ValidWELConfig", func(t *testing.T) {
+ config := &Configuration{}
+ config.Wel = usp_wel.WELConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "json",
+ },
+ EvtSources: "Security,System",
+ }
+
+ err := validateConfig("wel", config)
+ if err != nil {
+ t.Errorf("expected valid wel config to pass validation, got error: %v", err)
+ }
+ })
+
+ t.Run("InvalidWELConfigMissingIdentity", func(t *testing.T) {
+ config := &Configuration{}
+ config.Wel = usp_wel.WELConfig{
+ ClientOptions: uspclient.ClientOptions{},
+ EvtSources: "Security",
+ }
+
+ err := validateConfig("wel", config)
+ if err == nil {
+ t.Error("expected wel config with missing identity to fail validation")
+ }
+ })
+
+ t.Run("ValidFileConfig", func(t *testing.T) {
+ config := &Configuration{}
+ config.File = usp_file.FileConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "text",
+ },
+ FilePath: "/var/log/test.log",
+ }
+
+ err := validateConfig("file", config)
+ if err != nil {
+ t.Errorf("expected valid file config to pass validation, got error: %v", err)
+ }
+ })
+
+ t.Run("InvalidFileConfigMissingFilePath", func(t *testing.T) {
+ config := &Configuration{}
+ config.File = usp_file.FileConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "text",
+ },
+ // FilePath is missing
+ }
+
+ err := validateConfig("file", config)
+ if err == nil {
+ t.Error("expected file config with missing file_path to fail validation")
+ }
+ })
+
+ t.Run("InvalidFileConfigMissingIdentity", func(t *testing.T) {
+ config := &Configuration{}
+ config.File = usp_file.FileConfig{
+ ClientOptions: uspclient.ClientOptions{},
+ FilePath: "/var/log/test.log",
+ }
+
+ err := validateConfig("file", config)
+ if err == nil {
+ t.Error("expected file config with missing identity to fail validation")
+ }
+ })
+
+ t.Run("ValidK8sPodsConfig", func(t *testing.T) {
+ config := &Configuration{}
+ config.K8sPods = usp_k8s_pods.K8sPodsConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "json",
+ },
+ Root: "/var/log/pods",
+ }
+
+ err := validateConfig("k8s_pods", config)
+ if err != nil {
+ t.Errorf("expected valid k8s_pods config to pass validation, got error: %v", err)
+ }
+ })
+
+ t.Run("InvalidK8sPodsConfigMissingRoot", func(t *testing.T) {
+ config := &Configuration{}
+ config.K8sPods = usp_k8s_pods.K8sPodsConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "json",
+ },
+ // Root is missing
+ }
+
+ err := validateConfig("k8s_pods", config)
+ if err == nil {
+ t.Error("expected k8s_pods config with missing root to fail validation")
+ }
+ })
+
+ t.Run("InvalidK8sPodsConfigMissingIdentity", func(t *testing.T) {
+ config := &Configuration{}
+ config.K8sPods = usp_k8s_pods.K8sPodsConfig{
+ ClientOptions: uspclient.ClientOptions{},
+ Root: "/var/log/pods",
+ }
+
+ err := validateConfig("k8s_pods", config)
+ if err == nil {
+ t.Error("expected k8s_pods config with missing identity to fail validation")
+ }
+ })
+
+ t.Run("ValidBigQueryConfig", func(t *testing.T) {
+ config := &Configuration{}
+ config.BigQuery = usp_bigquery.BigQueryConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "json",
+ },
+ ProjectId: "my-project",
+ BigQueryProject: "my-project",
+ DatasetName: "my-dataset",
+ TableName: "my-table",
+ SqlQuery: "SELECT * FROM my-table",
+ }
+
+ err := validateConfig("bigquery", config)
+ if err != nil {
+ t.Errorf("expected valid bigquery config to pass validation, got error: %v", err)
+ }
+ })
+
+ t.Run("InvalidBigQueryConfigMissingProjectId", func(t *testing.T) {
+ config := &Configuration{}
+ config.BigQuery = usp_bigquery.BigQueryConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "json",
+ },
+ // ProjectId is missing
+ BigQueryProject: "my-project",
+ DatasetName: "my-dataset",
+ TableName: "my-table",
+ SqlQuery: "SELECT * FROM my-table",
+ }
+
+ err := validateConfig("bigquery", config)
+ if err == nil {
+ t.Error("expected bigquery config with missing project_id to fail validation")
+ }
+ })
+
+ t.Run("InvalidBigQueryConfigMissingSqlQuery", func(t *testing.T) {
+ config := &Configuration{}
+ config.BigQuery = usp_bigquery.BigQueryConfig{
+ ClientOptions: uspclient.ClientOptions{
+ Identity: uspclient.Identity{
+ Oid: "test-oid",
+ InstallationKey: "test-key",
+ },
+ Platform: "json",
+ },
+ ProjectId: "my-project",
+ BigQueryProject: "my-project",
+ DatasetName: "my-dataset",
+ TableName: "my-table",
+ // SqlQuery is missing
+ }
+
+ err := validateConfig("bigquery", config)
+ if err == nil {
+ t.Error("expected bigquery config with missing sql_query to fail validation")
+ }
+ })
+
+ t.Run("UnknownAdapterType", func(t *testing.T) {
+ config := &Configuration{}
+ err := validateConfig("unknown_adapter", config)
+ if err == nil {
+ t.Error("expected unknown adapter type to fail validation")
+ }
+ if err.Error() != "unknown adapter type: unknown_adapter" {
+ t.Errorf("expected specific error message, got: %v", err)
+ }
+ })
+
+ t.Run("AllKnownAdapterTypesSupported", func(t *testing.T) {
+ // List of all adapter types that should be supported
+ adapterTypes := []string{
+ "syslog", "pubsub", "gcs", "s3", "stdin", "1password", "bitwarden",
+ "itglue", "sophos", "okta", "office365", "wiz", "wel", "mac_unified_logging",
+ "azure_event_hub", "duo", "cato", "cylance", "entraid", "defender",
+ "slack", "sqs", "sqs-files", "simulator", "file", "evtx", "k8s_pods",
+ "bigquery", "imap", "hubspot", "falconcloud", "mimecast", "ms_graph",
+ "zendesk", "pandadoc", "proofpoint_tap", "box", "sublime",
+ "sentinel_one", "trendmicro",
+ }
+
+ config := &Configuration{}
+ for _, adapterType := range adapterTypes {
+ err := validateConfig(adapterType, config)
+ // We expect errors due to missing identity, but NOT "unknown adapter type"
+ if err != nil && err.Error() == "unknown adapter type: "+adapterType {
+ t.Errorf("adapter type %q should be supported but got unknown adapter error", adapterType)
+ }
+ }
+ })
+}
+
+// TestGetClientOptions tests the reflection-based getClientOptions function
+// that extracts ClientOptions from adapter configs based on json tags.
+func TestGetClientOptions(t *testing.T) {
+ t.Run("Syslog", func(t *testing.T) {
+ config := &Configuration{}
+ config.Syslog.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "test-oid-syslog"},
+ }
+
+ opts, err := getClientOptions("syslog", config)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if opts == nil {
+ t.Fatal("expected non-nil ClientOptions")
+ }
+ if opts.Identity.Oid != "test-oid-syslog" {
+ t.Errorf("expected Oid 'test-oid-syslog', got %q", opts.Identity.Oid)
+ }
+ })
+
+ t.Run("WEL", func(t *testing.T) {
+ config := &Configuration{}
+ config.Wel.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "test-oid-wel"},
+ }
+
+ opts, err := getClientOptions("wel", config)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if opts.Identity.Oid != "test-oid-wel" {
+ t.Errorf("expected Oid 'test-oid-wel', got %q", opts.Identity.Oid)
+ }
+ })
+
+ t.Run("S3", func(t *testing.T) {
+ config := &Configuration{}
+ config.S3.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "test-oid-s3"},
+ }
+
+ opts, err := getClientOptions("s3", config)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if opts.Identity.Oid != "test-oid-s3" {
+ t.Errorf("expected Oid 'test-oid-s3', got %q", opts.Identity.Oid)
+ }
+ })
+
+ t.Run("SqsFiles", func(t *testing.T) {
+ // Test adapter with hyphen in name (sqs-files)
+ config := &Configuration{}
+ config.SqsFiles.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "test-oid-sqs-files"},
+ }
+
+ opts, err := getClientOptions("sqs-files", config)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if opts.Identity.Oid != "test-oid-sqs-files" {
+ t.Errorf("expected Oid 'test-oid-sqs-files', got %q", opts.Identity.Oid)
+ }
+ })
+
+ t.Run("OnePassword", func(t *testing.T) {
+ // Test adapter with numeric prefix (1password)
+ config := &Configuration{}
+ config.OnePassword.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "test-oid-1password"},
+ }
+
+ opts, err := getClientOptions("1password", config)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if opts.Identity.Oid != "test-oid-1password" {
+ t.Errorf("expected Oid 'test-oid-1password', got %q", opts.Identity.Oid)
+ }
+ })
+
+ t.Run("UnknownAdapter", func(t *testing.T) {
+ config := &Configuration{}
+ _, err := getClientOptions("unknown_adapter", config)
+ if err == nil {
+ t.Fatal("expected error for unknown adapter")
+ }
+ if err.Error() != "unknown adapter type: unknown_adapter" {
+ t.Errorf("unexpected error message: %v", err)
+ }
+ })
+
+ t.Run("ReturnsPointerToActualField", func(t *testing.T) {
+ // Verify that modifying the returned pointer modifies the original config
+ config := &Configuration{}
+ config.Syslog.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "original"},
+ }
+
+ opts, err := getClientOptions("syslog", config)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ // Modify via the returned pointer
+ opts.Identity.Oid = "modified"
+
+ // Verify the original config was modified
+ if config.Syslog.ClientOptions.Identity.Oid != "modified" {
+ t.Error("expected modification via returned pointer to affect original config")
+ }
+ })
+
+ t.Run("AllKnownAdapterTypes", func(t *testing.T) {
+ // Comprehensive test that all adapter types work with reflection
+ adapterTypes := []string{
+ "syslog", "pubsub", "gcs", "s3", "stdin", "1password", "bitwarden",
+ "itglue", "sophos", "okta", "office365", "wiz", "wel", "mac_unified_logging",
+ "azure_event_hub", "duo", "cato", "cylance", "entraid", "defender",
+ "slack", "sqs", "sqs-files", "simulator", "file", "evtx", "k8s_pods",
+ "bigquery", "imap", "hubspot", "falconcloud", "mimecast", "ms_graph",
+ "zendesk", "pandadoc", "proofpoint_tap", "box", "sublime",
+ "sentinel_one", "trendmicro",
+ }
+
+ config := &Configuration{}
+ for _, adapterType := range adapterTypes {
+ opts, err := getClientOptions(adapterType, config)
+ if err != nil {
+ t.Errorf("adapter %q: unexpected error: %v", adapterType, err)
+ continue
+ }
+ if opts == nil {
+ t.Errorf("adapter %q: expected non-nil ClientOptions", adapterType)
+ }
+ }
+ })
+
+ t.Run("MatchesJsonTagNotFieldName", func(t *testing.T) {
+ // Verify reflection uses json tag, not Go field name
+ // Field name is "SentinelOne" but json tag is "sentinel_one"
+ config := &Configuration{}
+ config.SentinelOne.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "test-sentinel"},
+ }
+
+ // Should work with json tag name
+ opts, err := getClientOptions("sentinel_one", config)
+ if err != nil {
+ t.Fatalf("expected json tag 'sentinel_one' to work: %v", err)
+ }
+ if opts.Identity.Oid != "test-sentinel" {
+ t.Errorf("expected Oid 'test-sentinel', got %q", opts.Identity.Oid)
+ }
+
+ // Should NOT work with Go field name
+ _, err = getClientOptions("SentinelOne", config)
+ if err == nil {
+ t.Error("expected Go field name 'SentinelOne' to fail (should use json tag)")
+ }
+ })
+
+ t.Run("MultipleConfigsIndependent", func(t *testing.T) {
+ // Verify getting options for one adapter doesn't affect another
+ config := &Configuration{}
+ config.Syslog.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "syslog-oid"},
+ }
+ config.S3.ClientOptions = uspclient.ClientOptions{
+ Identity: uspclient.Identity{Oid: "s3-oid"},
+ }
+
+ syslogOpts, _ := getClientOptions("syslog", config)
+ s3Opts, _ := getClientOptions("s3", config)
+
+ if syslogOpts.Identity.Oid != "syslog-oid" {
+ t.Errorf("syslog Oid changed unexpectedly: %q", syslogOpts.Identity.Oid)
+ }
+ if s3Opts.Identity.Oid != "s3-oid" {
+ t.Errorf("s3 Oid changed unexpectedly: %q", s3Opts.Identity.Oid)
+ }
+ })
+}
+
+func TestConfigValidatorInterface(t *testing.T) {
+ t.Run("SyslogConfigImplementsInterface", func(t *testing.T) {
+ var _ ConfigValidator = &usp_syslog.SyslogConfig{}
+ })
+
+ t.Run("WELConfigImplementsInterface", func(t *testing.T) {
+ var _ ConfigValidator = &usp_wel.WELConfig{}
+ })
+
+ t.Run("FileConfigImplementsInterface", func(t *testing.T) {
+ var _ ConfigValidator = &usp_file.FileConfig{}
+ })
+
+ t.Run("K8sPodsConfigImplementsInterface", func(t *testing.T) {
+ var _ ConfigValidator = &usp_k8s_pods.K8sPodsConfig{}
+ })
+
+ t.Run("BigQueryConfigImplementsInterface", func(t *testing.T) {
+ var _ ConfigValidator = &usp_bigquery.BigQueryConfig{}
+ })
+}
+
+// TestCheckParsingResults tests the checkParsingResults function which validates
+// the response from the LimaCharlie USP parsing API.
+func TestCheckParsingResults(t *testing.T) {
+ t.Run("SuccessfulParsingWithResults", func(t *testing.T) {
+ // Successful parsing returns at least one event
+ result := &limacharlie.USPMappingValidationResponse{
+ Results: []limacharlie.Dict{
+ {"event_type": "INFO", "message": "test event 1"},
+ {"event_type": "WARN", "message": "test event 2"},
+ },
+ Errors: []string{},
+ }
+
+ err := checkParsingResults(result)
+ if err != nil {
+ t.Errorf("expected successful parsing to return nil, got error: %v", err)
+ }
+ })
+
+ t.Run("ParsingWithAPIErrors", func(t *testing.T) {
+ // API returns errors indicating parsing failure
+ result := &limacharlie.USPMappingValidationResponse{
+ Results: []limacharlie.Dict{},
+ Errors: []string{"regex pattern did not match", "invalid mapping configuration"},
+ }
+
+ err := checkParsingResults(result)
+ if err == nil {
+ t.Error("expected parsing with API errors to return error")
+ }
+ if !strings.Contains(err.Error(), "parsing validation failed") {
+ t.Errorf("expected error message to contain 'parsing validation failed', got: %v", err)
+ }
+ })
+
+ t.Run("EmptyResultsNoErrors", func(t *testing.T) {
+ // Empty results with no API errors - indicates regex doesn't match
+ // or wrong platform type
+ result := &limacharlie.USPMappingValidationResponse{
+ Results: []limacharlie.Dict{},
+ Errors: []string{},
+ }
+
+ err := checkParsingResults(result)
+ if err == nil {
+ t.Error("expected empty results to return error")
+ }
+ if !strings.Contains(err.Error(), "no events parsed") {
+ t.Errorf("expected error message to contain 'no events parsed', got: %v", err)
+ }
+ })
+
+ t.Run("NilResults", func(t *testing.T) {
+ // Nil results should be treated as empty results
+ result := &limacharlie.USPMappingValidationResponse{
+ Results: nil,
+ Errors: nil,
+ }
+
+ err := checkParsingResults(result)
+ if err == nil {
+ t.Error("expected nil results to return error")
+ }
+ if !strings.Contains(err.Error(), "no events parsed") {
+ t.Errorf("expected error message to contain 'no events parsed', got: %v", err)
+ }
+ })
+
+ t.Run("PartialParsingWithErrors", func(t *testing.T) {
+ // Some results but also some errors - errors take precedence
+ result := &limacharlie.USPMappingValidationResponse{
+ Results: []limacharlie.Dict{
+ {"event_type": "INFO", "message": "partial event"},
+ },
+ Errors: []string{"some lines failed to parse"},
+ }
+
+ err := checkParsingResults(result)
+ if err == nil {
+ t.Error("expected partial parsing with errors to return error")
+ }
+ if !strings.Contains(err.Error(), "parsing validation failed") {
+ t.Errorf("expected error message to contain 'parsing validation failed', got: %v", err)
+ }
+ })
+
+ t.Run("SingleEventSuccess", func(t *testing.T) {
+ // Single event is still a success
+ result := &limacharlie.USPMappingValidationResponse{
+ Results: []limacharlie.Dict{
+ {"event_type": "INFO"},
+ },
+ Errors: []string{},
+ }
+
+ err := checkParsingResults(result)
+ if err != nil {
+ t.Errorf("expected single event parsing to return nil, got error: %v", err)
+ }
+ })
+}
diff --git a/file/client.go b/file/client.go
index 258c264..e2ba0bd 100644
--- a/file/client.go
+++ b/file/client.go
@@ -5,7 +5,6 @@ package usp_file
import (
"context"
- "errors"
"fmt"
"io"
"os"
@@ -68,16 +67,6 @@ type FileAdapter struct {
lineCb func(line string) // callback for each line for testing
}
-func (c *FileConfig) Validate() error {
- if err := c.ClientOptions.Validate(); err != nil {
- return fmt.Errorf("client_options: %v", err)
- }
- if c.FilePath == "" {
- return errors.New("file_path missing")
- }
- return nil
-}
-
func NewFileAdapter(ctx context.Context, conf FileConfig) (*FileAdapter, chan struct{}, error) {
a := &FileAdapter{
conf: conf,
diff --git a/file/conf.go b/file/conf.go
index 6450e23..065b689 100644
--- a/file/conf.go
+++ b/file/conf.go
@@ -1,6 +1,9 @@
package usp_file
import (
+ "errors"
+ "fmt"
+
"github.com/refractionPOINT/go-uspclient"
)
@@ -16,3 +19,22 @@ type FileConfig struct {
Poll bool `json:"poll" yaml:"poll"`
MultiLineJSON bool `json:"multi_line_json" yaml:"multi_line_json"`
}
+
+// Validate validates the File adapter configuration.
+//
+// Parameters:
+//
+// None
+//
+// Returns:
+//
+// error - Returns nil if validation passes, or an error describing the validation failure.
+func (c *FileConfig) Validate() error {
+ if err := c.ClientOptions.Validate(); err != nil {
+ return fmt.Errorf("client_options: %v", err)
+ }
+ if c.FilePath == "" {
+ return errors.New("file_path missing")
+ }
+ return nil
+}
diff --git a/k8s_pods/client.go b/k8s_pods/client.go
index 8497bed..7fdf938 100644
--- a/k8s_pods/client.go
+++ b/k8s_pods/client.go
@@ -5,7 +5,6 @@ package usp_k8s_pods
import (
"context"
- "errors"
"fmt"
"regexp"
"sync"
@@ -36,16 +35,6 @@ type runtimeOptions struct {
excludePods *regexp.Regexp
}
-func (c *K8sPodsConfig) Validate() error {
- if err := c.ClientOptions.Validate(); err != nil {
- return fmt.Errorf("client_options: %v", err)
- }
- if c.Root == "" {
- return errors.New("file_path missing")
- }
- return nil
-}
-
func NewK8sPodsAdapter(ctx context.Context, conf K8sPodsConfig) (*K8sPodsAdapter, chan struct{}, error) {
a := &K8sPodsAdapter{
conf: conf,
diff --git a/k8s_pods/conf.go b/k8s_pods/conf.go
index 8cdd307..e8a962a 100644
--- a/k8s_pods/conf.go
+++ b/k8s_pods/conf.go
@@ -1,6 +1,9 @@
package usp_k8s_pods
import (
+ "errors"
+ "fmt"
+
"github.com/refractionPOINT/go-uspclient"
)
@@ -11,3 +14,22 @@ type K8sPodsConfig struct {
IncludePodsRE string `json:"include_pods_re" yaml:"include_pods_re"`
ExcludePodsRE string `json:"exclude_pods_re" yaml:"exclude_pods_re"`
}
+
+// Validate validates the K8s Pods adapter configuration.
+//
+// Parameters:
+//
+// None
+//
+// Returns:
+//
+// error - Returns nil if validation passes, or an error describing the validation failure.
+func (c *K8sPodsConfig) Validate() error {
+ if err := c.ClientOptions.Validate(); err != nil {
+ return fmt.Errorf("client_options: %v", err)
+ }
+ if c.Root == "" {
+ return errors.New("root missing")
+ }
+ return nil
+}