diff --git a/1password/client.go b/1password/client.go index 6599a80..af66b6d 100644 --- a/1password/client.go +++ b/1password/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -15,6 +14,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -31,15 +31,8 @@ type opRequest struct { Cursor string `json:"cursor,omitempty"` } -var URL = map[string]string{ - "business": "https://events.1password.com", - "enterprise": "https://events.ent.1password.com", - "ca": "https://events.1password.ca", - "eu": "https://events.1password.eu", -} - type OnePasswordAdapter struct { - conf OnePasswordConfig + conf adaptertypes.OnePasswordConfig uspClient *uspclient.Client httpClient *http.Client @@ -52,30 +45,7 @@ type OnePasswordAdapter struct { ctx context.Context } -type OnePasswordConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Token string `json:"token" yaml:"token"` - Endpoint string `json:"endpoint" yaml:"endpoint"` -} - -func (c *OnePasswordConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Token == "" { - return errors.New("missing token") - } - if c.Endpoint == "" { - return errors.New("missing endpoint") - } - _, ok := URL[c.Endpoint] - if !strings.HasPrefix(c.Endpoint, "https://") && !ok { - return fmt.Errorf("invalid endpoint, not https or in %v", URL) - } - return nil -} - -func NewOnePasswordpAdapter(conf OnePasswordConfig) (*OnePasswordAdapter, chan struct{}, error) { +func NewOnePasswordpAdapter(conf adaptertypes.OnePasswordConfig) (*OnePasswordAdapter, chan struct{}, error) { var err error a := &OnePasswordAdapter{ conf: conf, @@ -85,7 +55,7 @@ func NewOnePasswordpAdapter(conf OnePasswordConfig) (*OnePasswordAdapter, chan s if strings.HasPrefix(conf.Endpoint, "https://") { a.endpoint = conf.Endpoint - } else if v, ok := URL[conf.Endpoint]; ok { + } else if v, ok := adaptertypes.OnePasswordURL[conf.Endpoint]; ok { a.endpoint = v } else { return nil, nil, fmt.Errorf("not a valid api endpoint: %s", conf.Endpoint) diff --git a/adaptertypes/1password.go b/adaptertypes/1password.go new file mode 100644 index 0000000..7a89512 --- /dev/null +++ b/adaptertypes/1password.go @@ -0,0 +1,38 @@ +package adaptertypes + +import ( + "errors" + "fmt" + "strings" +) + +var OnePasswordURL = map[string]string{ + "business": "https://events.1password.com", + "enterprise": "https://events.ent.1password.com", + "ca": "https://events.1password.ca", + "eu": "https://events.1password.eu", +} + +// OnePasswordConfig defines the configuration for the 1Password adapter +type OnePasswordConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Token string `json:"token" yaml:"token" description:"1Password Events API bearer token" category:"auth" sensitive:"true" llmguidance:"Generate token in 1Password admin console under Integrations > Events Reporting"` + Endpoint string `json:"endpoint" yaml:"endpoint" description:"1Password API endpoint" category:"source" example:"business" llmguidance:"Options: 'business', 'enterprise', 'ca' (Canada), 'eu' (Europe). Or provide full HTTPS URL"` +} + +func (c *OnePasswordConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Token == "" { + return errors.New("missing token") + } + if c.Endpoint == "" { + return errors.New("missing endpoint") + } + _, ok := OnePasswordURL[c.Endpoint] + if !strings.HasPrefix(c.Endpoint, "https://") && !ok { + return fmt.Errorf("invalid endpoint, not https or in %v", OnePasswordURL) + } + return nil +} diff --git a/adaptertypes/azure_event_hub.go b/adaptertypes/azure_event_hub.go new file mode 100644 index 0000000..19ff3c7 --- /dev/null +++ b/adaptertypes/azure_event_hub.go @@ -0,0 +1,22 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// EventHubConfig defines the configuration for the Azure Event Hub adapter +type EventHubConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ConnectionString string `json:"connection_string" yaml:"connection_string" description:"Azure Event Hub connection string" category:"auth" sensitive:"true" llmguidance:"Found in Azure Portal under Event Hubs > Shared access policies. Requires 'Listen' permission"` +} + +func (c *EventHubConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ConnectionString == "" { + return errors.New("missing connection_string") + } + return nil +} diff --git a/adaptertypes/bigquery.go b/adaptertypes/bigquery.go new file mode 100644 index 0000000..9ab579e --- /dev/null +++ b/adaptertypes/bigquery.go @@ -0,0 +1,38 @@ +package adaptertypes + +import ( + "errors" +) + +// BigQueryConfig defines the configuration for the Google BigQuery adapter +type BigQueryConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ProjectId string `json:"project_id" yaml:"project_id" description:"GCP project ID for billing/quota" category:"source" example:"my-project-123"` + BigQueryProject string `json:"bigquery_project" yaml:"bigquery_project" description:"GCP project containing the BigQuery dataset" category:"source" example:"analytics-project"` + DatasetName string `json:"dataset_name" yaml:"dataset_name" description:"BigQuery dataset name" category:"source" example:"logs_dataset"` + TableName string `json:"table_name" yaml:"table_name" description:"BigQuery table name" category:"source" example:"security_events"` + ServiceAccountCreds string `json:"service_account_creds,omitempty" yaml:"service_account_creds,omitempty" description:"GCP service account JSON credentials" category:"auth" sensitive:"true"` + SqlQuery string `json:"sql_query" yaml:"sql_query" description:"SQL query to execute" category:"source" llmguidance:"Use WHERE clause to filter data. Include timestamp column for incremental queries"` + QueryInterval string `json:"query_interval" yaml:"query_interval" description:"Interval between query executions" category:"behavior" example:"1h" default:"1h" llmguidance:"Format: duration string like '1h', '30m', '24h'"` + IsOneTimeLoad bool `json:"is_one_time_load" yaml:"is_one_time_load" description:"If true, run query once and exit. If false, run continuously at intervals" category:"behavior" default:"false"` +} + +func (bq *BigQueryConfig) Validate() error { + if bq.ProjectId == "" { + return errors.New("missing project_id") + } + // this will usually be th same as projectID but could be different if using outside project dataset such as a public data set + if bq.BigQueryProject == "" { + return errors.New("missing bigquery project name") + } + if bq.DatasetName == "" { + return errors.New("missing dataset_name") + } + if bq.TableName == "" { + return errors.New("missing table_name") + } + if bq.SqlQuery == "" { + return errors.New("missing sql query") + } + return nil +} diff --git a/adaptertypes/bitwarden.go b/adaptertypes/bitwarden.go new file mode 100644 index 0000000..fd5f19b --- /dev/null +++ b/adaptertypes/bitwarden.go @@ -0,0 +1,55 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// BitwardenConfig defines the configuration for the Bitwarden adapter +type BitwardenConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ClientID string `json:"client_id" yaml:"client_id" description:"Bitwarden API client ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Bitwarden API client secret" category:"auth" sensitive:"true"` + Region string `json:"region" yaml:"region" description:"Bitwarden region" category:"source" example:"US" llmguidance:"Options: 'US' or 'EU'"` + TokenEndpointURL string `json:"token_endpoint_url" yaml:"token_endpoint_url" description:"Custom token endpoint URL" category:"source" llmguidance:"Optional. Override default token endpoint"` + EventsBaseURL string `json:"events_base_url" yaml:"events_base_url" description:"Custom events API base URL" category:"source" llmguidance:"Optional. Override default events endpoint"` +} + +func (c *BitwardenConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + + if c.ClientID == "" { + return errors.New("missing client_id") + } + if c.ClientSecret == "" { + return errors.New("missing client_secret") + } + + // Check if custom URLs are provided + hasCustomURLs := c.TokenEndpointURL != "" || c.EventsBaseURL != "" + + if hasCustomURLs { + // If custom URLs are provided, both must be set and region must be empty + if c.TokenEndpointURL == "" { + return errors.New("token_endpoint_url must be set when events_base_url is provided") + } + if c.EventsBaseURL == "" { + return errors.New("events_base_url must be set when token_endpoint_url is provided") + } + if c.Region != "" { + return errors.New("region cannot be set when using custom URLs (token_endpoint_url and events_base_url)") + } + } else { + // If custom URLs are not provided, use region-based configuration + if c.Region == "" { + c.Region = "us" + } + if c.Region != "us" && c.Region != "eu" { + return fmt.Errorf("invalid region: %s (must be 'us' or 'eu')", c.Region) + } + } + + return nil +} diff --git a/adaptertypes/box.go b/adaptertypes/box.go new file mode 100644 index 0000000..a95bbea --- /dev/null +++ b/adaptertypes/box.go @@ -0,0 +1,25 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// BoxConfig defines the configuration for the Box adapter +type BoxConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ClientID string `json:"client_id" yaml:"client_id" description:"Box OAuth2 client ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Box OAuth2 client secret" category:"auth" sensitive:"true"` + SubjectID string `json:"subject_id" yaml:"subject_id" description:"Box enterprise or user ID" category:"auth" llmguidance:"Enterprise ID for enterprise events, or user ID for user events"` +} + +func (c *BoxConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + + if c.ClientID == "" || c.ClientSecret == "" || c.SubjectID == "" { + return errors.New("missing Box client ID, secret, or subject ID") + } + return nil +} diff --git a/adaptertypes/cato.go b/adaptertypes/cato.go new file mode 100644 index 0000000..fda921c --- /dev/null +++ b/adaptertypes/cato.go @@ -0,0 +1,27 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// CatoConfig defines the configuration for the Cato Networks adapter +type CatoConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` + ApiKey string `json:"apikey" yaml:"apikey" description:"Cato Networks API key" category:"auth" sensitive:"true"` + AccountId int `json:"accountid" yaml:"accountid" description:"Cato Networks account ID" category:"source"` +} + +func (c *CatoConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.AccountId == 0 { + return errors.New("missing account id") + } + if c.ApiKey == "" { + return errors.New("missing api key") + } + return nil +} diff --git a/adaptertypes/cylance.go b/adaptertypes/cylance.go new file mode 100644 index 0000000..6b2e47f --- /dev/null +++ b/adaptertypes/cylance.go @@ -0,0 +1,36 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +const defaultLoggingBaseURL = "https://protectapi.cylance.com" + +// CylanceConfig defines the configuration for the Cylance adapter +type CylanceConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + TenantID string `json:"tenant_id" yaml:"tenant_id" description:"Cylance tenant ID" category:"auth"` + AppID string `json:"app_id" yaml:"app_id" description:"Cylance application ID" category:"auth"` + AppSecret string `json:"app_secret" yaml:"app_secret" description:"Cylance application secret" category:"auth" sensitive:"true"` + LoggingBaseURL string `json:"logging_base_url" yaml:"logging_base_url" description:"Cylance logging API base URL" category:"source" example:"https://protectapi.cylance.com"` +} + +func (c *CylanceConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.TenantID == "" { + return errors.New("missing tenant id") + } + if c.AppID == "" { + return errors.New("missing app id") + } + if c.AppSecret == "" { + return errors.New("missing app secret") + } + if c.LoggingBaseURL == "" { + c.LoggingBaseURL = defaultLoggingBaseURL + } + return nil +} diff --git a/adaptertypes/defender.go b/adaptertypes/defender.go new file mode 100644 index 0000000..8a64e3d --- /dev/null +++ b/adaptertypes/defender.go @@ -0,0 +1,30 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// DefenderConfig defines the configuration for the Microsoft Defender adapter +type DefenderConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + TenantID string `json:"tenant_id" yaml:"tenant_id" description:"Azure AD tenant ID" category:"auth" example:"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"` + ClientID string `json:"client_id" yaml:"client_id" description:"Azure AD application (client) ID" category:"auth" example:"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Azure AD application client secret" category:"auth" sensitive:"true"` +} + +func (c *DefenderConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.TenantID == "" { + return errors.New("missing tenant_id") + } + if c.ClientID == "" { + return errors.New("missing client_id") + } + if c.ClientSecret == "" { + return errors.New("missing client_secret") + } + return nil +} diff --git a/adaptertypes/duo.go b/adaptertypes/duo.go new file mode 100644 index 0000000..dc8b8b7 --- /dev/null +++ b/adaptertypes/duo.go @@ -0,0 +1,30 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// DuoConfig defines the configuration for the Duo Security adapter +type DuoConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + IntegrationKey string `json:"integration_key" yaml:"integration_key" description:"Duo Admin API integration key" category:"auth" sensitive:"true" llmguidance:"Generate in Duo Admin Panel > Applications > Admin API"` + SecretKey string `json:"secret_key" yaml:"secret_key" description:"Duo Admin API secret key" category:"auth" sensitive:"true"` + APIHostname string `json:"api_hostname" yaml:"api_hostname" description:"Duo API hostname" category:"source" example:"api-xxxxx.duosecurity.com" llmguidance:"Found in Duo Admin Panel under your Admin API application"` +} + +func (c *DuoConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.IntegrationKey == "" { + return errors.New("missing integration_key") + } + if c.SecretKey == "" { + return errors.New("missing secret_key") + } + if c.APIHostname == "" { + return errors.New("missing api_hostname") + } + return nil +} diff --git a/adaptertypes/entraid.go b/adaptertypes/entraid.go new file mode 100644 index 0000000..20891b7 --- /dev/null +++ b/adaptertypes/entraid.go @@ -0,0 +1,30 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// EntraIDConfig defines the configuration for the Microsoft Entra ID (Azure AD) adapter +type EntraIDConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + TenantID string `json:"tenant_id" yaml:"tenant_id" description:"Azure AD tenant ID" category:"auth" example:"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"` + ClientID string `json:"client_id" yaml:"client_id" description:"Azure AD application (client) ID" category:"auth" example:"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Azure AD application client secret" category:"auth" sensitive:"true"` +} + +func (c *EntraIDConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.TenantID == "" { + return errors.New("missing tenant_id") + } + if c.ClientID == "" { + return errors.New("missing client_id") + } + if c.ClientSecret == "" { + return errors.New("missing client_secret") + } + return nil +} diff --git a/adaptertypes/evtx.go b/adaptertypes/evtx.go new file mode 100644 index 0000000..33868ad --- /dev/null +++ b/adaptertypes/evtx.go @@ -0,0 +1,23 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// EVTXConfig defines the configuration for the EVTX file parser adapter +type EVTXConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` + FilePath string `json:"file_path" yaml:"file_path" description:"Path to the EVTX file to parse" category:"source" example:"C:\\Windows\\System32\\winevt\\Logs\\Security.evtx" llmguidance:"For Windows Event Log files (.evtx format). Use for one-time parsing of archived event logs"` +} + +func (c *EVTXConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.FilePath == "" { + return errors.New("file_path missing") + } + return nil +} diff --git a/adaptertypes/falconcloud.go b/adaptertypes/falconcloud.go new file mode 100644 index 0000000..c0f2620 --- /dev/null +++ b/adaptertypes/falconcloud.go @@ -0,0 +1,31 @@ +package adaptertypes + +import ( + "errors" + "fmt" + "time" +) + +// FalconCloudConfig defines the configuration for the CrowdStrike Falcon adapter +type FalconCloudConfig struct { + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ClientId string `json:"client_id" yaml:"client_id" description:"CrowdStrike API client ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"CrowdStrike API client secret" category:"auth" sensitive:"true"` + IsUsingOffset bool `json:"is_using_offset" yaml:"is_using_offset" description:"Use offset-based pagination instead of time-based" category:"behavior" default:"false"` + Offset uint64 `json:"offset" yaml:"offset" description:"Starting offset for event stream" category:"behavior" default:"0"` + NotBefore *time.Time `json:"not_before,omitempty" yaml:"not_before,omitempty" description:"Only fetch events after this timestamp" category:"behavior"` +} + +func (c *FalconCloudConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ClientId == "" { + return errors.New("missing client id") + } + if c.ClientSecret == "" { + return errors.New("missing client secret") + } + return nil +} diff --git a/adaptertypes/file.go b/adaptertypes/file.go new file mode 100644 index 0000000..4ddcef3 --- /dev/null +++ b/adaptertypes/file.go @@ -0,0 +1,39 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// FileConfig defines the configuration for the file adapter +type FileConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"120" llmguidance:"Increase if processing very large files or slow network connections"` + + FilePath string `json:"file_path" yaml:"file_path" description:"Path to the log file or directory to monitor" category:"source" example:"/var/log/application/*.log" llmguidance:"Supports wildcards (*). For directories, specify pattern like '/var/log/*.log'. For single file, specify exact path"` + + NoFollow bool `json:"no_follow" yaml:"no_follow" description:"If true, do not follow file as it grows (read once and exit)" category:"behavior" default:"false" llmguidance:"Set to true for one-time file reads. Set to false to continuously tail the file (like 'tail -f')"` + + InactivityThreshold int `json:"inactivity_threshold" yaml:"inactivity_threshold" description:"Seconds of inactivity before marking file as inactive" category:"behavior" default:"300" llmguidance:"Used to detect when log files have been rotated or are no longer being written to"` + + ReactivationThreshold int `json:"reactivation_threshold" yaml:"reactivation_threshold" description:"Minimum number of new bytes before reactivating an inactive file" category:"behavior" default:"100" llmguidance:"Prevents reactivation on small writes. Set lower if logs are written infrequently"` + + Backfill bool `json:"backfill" yaml:"backfill" description:"If true, read existing file contents before tailing. If false, only read new data" category:"behavior" default:"false" llmguidance:"Set to true to ingest historical data from existing files. Set to false to only capture new log entries written after adapter starts"` + + SerializeFiles bool `json:"serialize_files" yaml:"serialize_files" description:"If true, process files one at a time. If false, process multiple files concurrently" category:"performance" default:"false" llmguidance:"Set to true if file processing order matters or to reduce resource usage"` + + Poll bool `json:"poll" yaml:"poll" description:"If true, use polling instead of inotify/fsevents for file changes" category:"behavior" default:"false" llmguidance:"Enable if running in environments where file system notifications don't work (NFS, some containers)"` + + MultiLineJSON bool `json:"multi_line_json" yaml:"multi_line_json" description:"If true, parse multi-line formatted JSON objects" category:"parsing" default:"false" llmguidance:"Enable if JSON objects span multiple lines (pretty-printed JSON). Disable for line-delimited JSON (JSONL/NDJSON)"` +} + +func (c *FileConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.FilePath == "" { + return errors.New("file_path missing") + } + return nil +} diff --git a/adaptertypes/gcs.go b/adaptertypes/gcs.go new file mode 100644 index 0000000..5353d3e --- /dev/null +++ b/adaptertypes/gcs.go @@ -0,0 +1,26 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// GCSConfig defines the configuration for the Google Cloud Storage adapter +type GCSConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + BucketName string `json:"bucket_name" yaml:"bucket_name" description:"Name of the GCS bucket containing log files" category:"source" example:"my-logs-bucket" llmguidance:"Provide just the bucket name, not gs:// URI. Example: 'my-company-logs'"` + ServiceAccountCreds string `json:"service_account_creds,omitempty" yaml:"service_account_creds,omitempty" description:"GCP service account JSON credentials" category:"auth" sensitive:"true" llmguidance:"Provide the full JSON key file contents as a string. Service account needs roles/storage.objectViewer and roles/storage.objectAdmin (for deletion) permissions"` + IsOneTimeLoad bool `json:"single_load" yaml:"single_load" description:"If true, loads all files once without deletion. If false, continuously monitors bucket and deletes files after successful ingestion" category:"behavior" default:"false" llmguidance:"Set to true for one-time historical data imports. Set to false for continuous live monitoring. When false, files are deleted after processing"` + Prefix string `json:"prefix" yaml:"prefix" description:"Object prefix to filter which objects to process" category:"source" example:"logs/application/" llmguidance:"Optional. Use to limit processing to specific paths within the bucket. Leave empty to process all objects"` + ParallelFetch int `json:"parallel_fetch" yaml:"parallel_fetch" description:"Number of parallel file downloads from GCS" category:"performance" default:"1" llmguidance:"Increase for better throughput. Recommended: 5-10 for most use cases. Higher values use more memory"` +} + +func (c *GCSConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.BucketName == "" { + return errors.New("missing bucket_name") + } + return nil +} diff --git a/adaptertypes/go.mod b/adaptertypes/go.mod new file mode 100644 index 0000000..43da5d0 --- /dev/null +++ b/adaptertypes/go.mod @@ -0,0 +1,74 @@ +module github.com/refractionPOINT/usp-adapters/adaptertypes + +go 1.24.0 + +toolchain go1.24.10 + +require ( + github.com/refractionPOINT/go-uspclient v1.5.1 + github.com/refractionPOINT/usp-adapters v1.32.4 +) + +require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.4 // indirect + cloud.google.com/go/auth v0.16.3 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + cloud.google.com/go/storage v1.56.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/elastic/go-grok v0.3.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/magefile/mage v1.15.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20250728014624-2a3d8e58cb31 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.37.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect + golang.org/x/time v0.12.0 // indirect + google.golang.org/api v0.244.0 // indirect + google.golang.org/genproto v0.0.0-20250728155136-f173205681a0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect + google.golang.org/grpc v1.74.2 // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/adaptertypes/go.sum b/adaptertypes/go.sum new file mode 100644 index 0000000..4077b9d --- /dev/null +++ b/adaptertypes/go.sum @@ -0,0 +1,168 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.4 h1:cVvUiY0sX0xwyxPwdSU2KsF9knOVmtRyAMt8xou0iTs= +cloud.google.com/go v0.121.4/go.mod h1:XEBchUiHFJbz4lKBZwYBDHV/rSyfFktk737TLDU089s= +cloud.google.com/go/auth v0.16.3 h1:kabzoQ9/bobUmnseYnBO6qQG7q4a/CffFRlJSxv2wCc= +cloud.google.com/go/auth v0.16.3/go.mod h1:NucRGjaXfzP1ltpcQ7On/VTZ0H4kWB5Jy+Y9Dnm76fA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= +cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= +github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20250728014624-2a3d8e58cb31 h1:+oE33jyVrqgxhH1c68jEEhNmrgxWC1qI8dTiE1pxb1A= +github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20250728014624-2a3d8e58cb31/go.mod h1:CZY5ZoKVjNsKt2g9YGkx1qE5ifHdcI7uB00bx37zaNI= +github.com/refractionPOINT/go-uspclient v1.5.1 h1:/v+UFW1OqcBMuUjztq9Og3WQRIvB2H/hxP9wfvrxib8= +github.com/refractionPOINT/go-uspclient v1.5.1/go.mod h1:j4tCnHNLMcx80nZ2KHKLGgLFPW2UVvFVxRil6Qd6Wjo= +github.com/refractionPOINT/usp-adapters v1.32.4 h1:N+x6nwABlefrMECJX3vAEpSDQ2ulArhSvzXvQ2DtK2I= +github.com/refractionPOINT/usp-adapters v1.32.4/go.mod h1:dvDNZXuUFawNRzpfwxHZHUXnc095CBnVLFwTrvFv7l4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.37.0 h1:B+WbN9RPsvobe6q4vP6KgM8/9plR/HNjgGBrfcOlweA= +go.opentelemetry.io/contrib/detectors/gcp v1.37.0/go.mod h1:K5zQ3TT7p2ru9Qkzk0bKtCql0RGkPj9pRjpXgZJZ+rU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/api v0.244.0 h1:lpkP8wVibSKr++NCD36XzTk/IzeKJ3klj7vbj+XU5pE= +google.golang.org/api v0.244.0/go.mod h1:dMVhVcylamkirHdzEBAIQWUCgqY885ivNeZYd7VAVr8= +google.golang.org/genproto v0.0.0-20250728155136-f173205681a0 h1:btBcgujH2+KIWEfz0s7Cdtt9R7hpwM4SAEXAdXf/ddw= +google.golang.org/genproto v0.0.0-20250728155136-f173205681a0/go.mod h1:Q4yZQ3kmmIyg6HsMjCGx2vQ8gzN+dntaPmFWz6Zj0fo= +google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0 h1:0UOBWO4dC+e51ui0NFKSPbkHHiQ4TmrEfEZMLDyRmY8= +google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0/go.mod h1:8ytArBbtOy2xfht+y2fqKd5DRDJRUQhqbyEnQ4bDChs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 h1:MAKi5q709QWfnkkpNQ0M12hYJ1+e8qYVDyowc4U1XZM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/adaptertypes/hubspot.go b/adaptertypes/hubspot.go new file mode 100644 index 0000000..a8f19b7 --- /dev/null +++ b/adaptertypes/hubspot.go @@ -0,0 +1,22 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// HubSpotConfig defines the configuration for the HubSpot adapter +type HubSpotConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + AccessToken string `json:"access_token" yaml:"access_token" description:"HubSpot private app access token" category:"auth" sensitive:"true" llmguidance:"Generate in HubSpot Settings > Integrations > Private Apps"` +} + +func (c *HubSpotConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.AccessToken == "" { + return errors.New("missing access token") + } + return nil +} diff --git a/adaptertypes/imap.go b/adaptertypes/imap.go new file mode 100644 index 0000000..6d9aefb --- /dev/null +++ b/adaptertypes/imap.go @@ -0,0 +1,37 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// ImapConfig defines the configuration for the IMAP email adapter +type ImapConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Server string `json:"server" yaml:"server" description:"IMAP server address with port" category:"source" example:"imap.gmail.com:993"` + UserName string `json:"username" yaml:"username" description:"IMAP username/email" category:"auth"` + Password string `json:"password" yaml:"password" description:"IMAP password or app password" category:"auth" sensitive:"true"` + InboxName string `json:"inbox_name" yaml:"inbox_name" description:"IMAP folder/inbox name to monitor" category:"source" default:"INBOX"` + IsInsecure bool `json:"is_insecure" yaml:"is_insecure" description:"Skip TLS certificate verification" category:"behavior" default:"false" llmguidance:"Only enable for testing. Keep false for production"` + FromZero bool `json:"from_zero" yaml:"from_zero" description:"Start from first email in inbox" category:"behavior" default:"false" llmguidance:"If true, process all emails. If false, only process new emails"` + IncludeAttachments bool `json:"include_attachments" yaml:"include_attachments" description:"Extract and include email attachments" category:"parsing" default:"false"` + MaxBodySize int `json:"max_body_size" yaml:"max_body_size" description:"Maximum email body size in bytes" category:"parsing" default:"1048576" llmguidance:"Default 1MB. Increase for large emails"` + AttachmentIngestKey string `json:"attachment_ingest_key" yaml:"attachment_ingest_key" description:"LimaCharlie ingestion key for uploading attachments" category:"client"` + AttachmentRetentionDays int `json:"attachment_retention_days" yaml:"attachment_retention_days" description:"Days to retain uploaded attachments" category:"client" default:"30"` +} + +func (c *ImapConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Server == "" { + return errors.New("missing server") + } + if c.UserName == "" { + return errors.New("missing username") + } + if c.Password == "" { + return errors.New("missing password") + } + return nil +} diff --git a/adaptertypes/itglue.go b/adaptertypes/itglue.go new file mode 100644 index 0000000..82dd125 --- /dev/null +++ b/adaptertypes/itglue.go @@ -0,0 +1,22 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// ITGlueConfig defines the configuration for the IT Glue adapter +type ITGlueConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Token string `json:"token" yaml:"token" description:"IT Glue API token" category:"auth" sensitive:"true"` +} + +func (c *ITGlueConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Token == "" { + return errors.New("missing token") + } + return nil +} diff --git a/adaptertypes/k8s_pods.go b/adaptertypes/k8s_pods.go new file mode 100644 index 0000000..19bbfc6 --- /dev/null +++ b/adaptertypes/k8s_pods.go @@ -0,0 +1,25 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// K8sPodsConfig defines the configuration for the Kubernetes pods log adapter +type K8sPodsConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` + Root string `json:"root" yaml:"root" description:"Root directory containing pod logs" category:"source" example:"/var/log/pods" llmguidance:"Usually /var/log/pods on the node filesystem"` + IncludePodsRE string `json:"include_pods_re" yaml:"include_pods_re" description:"Regular expression to include specific pods" category:"filter" llmguidance:"Optional. Include only pods matching this regex"` + ExcludePodsRE string `json:"exclude_pods_re" yaml:"exclude_pods_re" description:"Regular expression to exclude specific pods" category:"filter" llmguidance:"Optional. Exclude pods matching this regex"` +} + +func (c *K8sPodsConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Root == "" { + return errors.New("file_path missing") + } + return nil +} diff --git a/adaptertypes/mac_unified_logging.go b/adaptertypes/mac_unified_logging.go new file mode 100644 index 0000000..9792c29 --- /dev/null +++ b/adaptertypes/mac_unified_logging.go @@ -0,0 +1,14 @@ +package adaptertypes + +// MacUnifiedLoggingConfig defines the configuration for the macOS Unified Logging adapter +type MacUnifiedLoggingConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` + Predicate string `json:"predicate,omitempty" yaml:"predicate,omitempty" description:"macOS log predicate filter" category:"source" example:"subsystem == 'com.apple.securityd'" llmguidance:"Optional. Filter logs using NSPredicate syntax. Leave empty for all logs"` +} + +// Validate validates the MacUnifiedLoggingConfig +// Note: This config doesn't have required fields beyond ClientOptions +func (c *MacUnifiedLoggingConfig) Validate() error { + return c.ClientOptions.Validate() +} diff --git a/adaptertypes/mimecast.go b/adaptertypes/mimecast.go new file mode 100644 index 0000000..f01c4d9 --- /dev/null +++ b/adaptertypes/mimecast.go @@ -0,0 +1,27 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// MimecastConfig defines the configuration for the Mimecast adapter +type MimecastConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ClientId string `json:"client_id" yaml:"client_id" description:"Mimecast API client ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Mimecast API client secret" category:"auth" sensitive:"true"` +} + +func (c *MimecastConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ClientId == "" { + return errors.New("missing client id") + } + if c.ClientSecret == "" { + return errors.New("missing client secret") + } + + return nil +} diff --git a/adaptertypes/ms_graph.go b/adaptertypes/ms_graph.go new file mode 100644 index 0000000..6a7c4f8 --- /dev/null +++ b/adaptertypes/ms_graph.go @@ -0,0 +1,34 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// MsGraphConfig defines the configuration for the Microsoft Graph adapter +type MsGraphConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + TenantID string `json:"tenant_id" yaml:"tenant_id" description:"Azure AD tenant ID" category:"auth"` + ClientID string `json:"client_id" yaml:"client_id" description:"Azure AD application (client) ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Azure AD application client secret" category:"auth" sensitive:"true"` + URL string `json:"url" yaml:"url" description:"Microsoft Graph API URL to query" category:"source" example:"https://graph.microsoft.com/v1.0/security/alerts" llmguidance:"Full Graph API endpoint URL"` +} + +func (c *MsGraphConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.TenantID == "" { + return errors.New("missing tenant_id") + } + if c.ClientID == "" { + return errors.New("missing client_id") + } + if c.ClientSecret == "" { + return errors.New("missing client_secret") + } + if c.URL == "" { + return errors.New("missing url") + } + return nil +} diff --git a/adaptertypes/office365.go b/adaptertypes/office365.go new file mode 100644 index 0000000..4457221 --- /dev/null +++ b/adaptertypes/office365.go @@ -0,0 +1,59 @@ +package adaptertypes + +import ( + "errors" + "fmt" + "strings" +) + +// Office365Config defines the configuration for the Office 365 adapter +type Office365Config struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Domain string `json:"domain" yaml:"domain" description:"Office 365 domain" category:"source" example:"contoso.com"` + TenantID string `json:"tenant_id" yaml:"tenant_id" description:"Azure AD tenant ID" category:"auth"` + PublisherID string `json:"publisher_id" yaml:"publisher_id" description:"Publisher ID for Office 365 Management API" category:"auth"` + ClientID string `json:"client_id" yaml:"client_id" description:"Azure AD application (client) ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Azure AD application client secret" category:"auth" sensitive:"true"` + Endpoint string `json:"endpoint" yaml:"endpoint" description:"Office 365 endpoint" category:"source" example:"commercial" llmguidance:"Options: 'commercial', 'gcc', 'gcchigh', 'dod'"` + ContentTypes string `json:"content_types" yaml:"content_types" description:"Comma-separated list of content types to fetch" category:"source" example:"Audit.AzureActiveDirectory,Audit.Exchange" llmguidance:"Common: Audit.AzureActiveDirectory, Audit.Exchange, Audit.SharePoint, Audit.General"` + StartTime string `json:"start_time" yaml:"start_time" description:"Start time for initial fetch" category:"behavior" llmguidance:"RFC3339 format. Leave empty to start from current time"` + + Deduper Deduper `json:"-" yaml:"-"` +} + +// Office 365 Management API endpoints for different cloud environments +var office365URL = map[string]string{ + "enterprise": "https://manage.office.com/api/v1.0/", + "gcc-gov": "https://manage-gcc.office.com/api/v1.0/", + "gcc-high-gov": "https://manage.office365.us/api/v1.0/", + "dod-gov": "https://manage.protection.apps.mil/api/v1.0/", +} + +func (c *Office365Config) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Domain == "" { + return errors.New("missing domain") + } + if c.TenantID == "" { + return errors.New("missing tenant_id") + } + if c.PublisherID == "" { + return errors.New("missing publisher_id") + } + if c.ClientID == "" { + return errors.New("missing client_id") + } + if c.ClientSecret == "" { + return errors.New("missing client_secret") + } + if c.Endpoint == "" { + return errors.New("missing endpoint") + } + _, ok := office365URL[c.Endpoint] + if !strings.HasPrefix(c.Endpoint, "https://") && !ok { + return fmt.Errorf("invalid endpoint, not https or in %v", office365URL) + } + return nil +} diff --git a/adaptertypes/okta.go b/adaptertypes/okta.go new file mode 100644 index 0000000..1498786 --- /dev/null +++ b/adaptertypes/okta.go @@ -0,0 +1,28 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// OktaConfig defines the configuration for the Okta adapter +type OktaConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + + ApiKey string `json:"apikey" yaml:"apikey" description:"Okta API token for authentication" category:"auth" sensitive:"true" llmguidance:"Generate an API token in Okta Admin Console > Security > API > Tokens. Requires read permissions for system log events"` + + URL string `json:"url" yaml:"url" description:"Okta organization URL" category:"source" example:"https://your-org.okta.com" llmguidance:"Your Okta organization domain. Format: https://.okta.com (no trailing slash)"` +} + +func (c *OktaConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.URL == "" { + return errors.New("missing url") + } + if c.ApiKey == "" { + return errors.New("missing api key") + } + return nil +} diff --git a/adaptertypes/pandadoc.go b/adaptertypes/pandadoc.go new file mode 100644 index 0000000..c6f0579 --- /dev/null +++ b/adaptertypes/pandadoc.go @@ -0,0 +1,23 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// PandaDocConfig defines the configuration for the PandaDoc adapter +type PandaDocConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ApiKey string `json:"api_key" yaml:"api_key" description:"PandaDoc API key" category:"auth" sensitive:"true"` +} + +func (c *PandaDocConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ApiKey == "" { + return errors.New("missing api key") + } + + return nil +} diff --git a/adaptertypes/proofpoint_tap.go b/adaptertypes/proofpoint_tap.go new file mode 100644 index 0000000..e3444bc --- /dev/null +++ b/adaptertypes/proofpoint_tap.go @@ -0,0 +1,26 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// ProofpointTapConfig defines the configuration for the Proofpoint TAP adapter +type ProofpointTapConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Principal string `json:"principal" yaml:"principal" description:"Proofpoint TAP service principal" category:"auth"` + Secret string `json:"secret" yaml:"secret" description:"Proofpoint TAP service secret" category:"auth" sensitive:"true"` +} + +func (c *ProofpointTapConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Principal == "" { + return errors.New("missing principal") + } + if c.Secret == "" { + return errors.New("missing secret") + } + return nil +} diff --git a/adaptertypes/pubsub.go b/adaptertypes/pubsub.go new file mode 100644 index 0000000..3579a0c --- /dev/null +++ b/adaptertypes/pubsub.go @@ -0,0 +1,31 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// PubSubConfig defines the configuration for the Google Cloud Pub/Sub adapter +type PubSubConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + SubscriptionName string `json:"sub_name" yaml:"sub_name" description:"Name of the Pub/Sub subscription to consume from" category:"source" example:"my-logs-subscription" llmguidance:"The subscription must already exist. Format: 'subscription-name' or 'projects/PROJECT_ID/subscriptions/SUBSCRIPTION_NAME'"` + ProjectName string `json:"project_name" yaml:"project_name" description:"GCP project ID" category:"source" example:"my-gcp-project" llmguidance:"The GCP project ID (not project name or number)"` + ServiceAccountCreds string `json:"service_account_creds,omitempty" yaml:"service_account_creds,omitempty" description:"GCP service account JSON credentials" category:"auth" sensitive:"true" llmguidance:"Provide full JSON key file contents. Requires pubsub.subscriber role. Leave empty to use default service account"` + MaxPSBuffer int `json:"max_ps_buffer,omitempty" yaml:"max_ps_buffer,omitempty" description:"Maximum number of messages to buffer" category:"performance" default:"1000"` +} + +func (c *PubSubConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.SubscriptionName == "" { + return errors.New("missing sub_name") + } + if c.ProjectName == "" { + return errors.New("missing project_name") + } + if c.ServiceAccountCreds == "" { + return errors.New("missing service_account_creds") + } + return nil +} diff --git a/adaptertypes/s3.go b/adaptertypes/s3.go new file mode 100644 index 0000000..58bfb96 --- /dev/null +++ b/adaptertypes/s3.go @@ -0,0 +1,41 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// S3Config defines the configuration for the Amazon S3 adapter +type S3Config struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + + BucketName string `json:"bucket_name" yaml:"bucket_name" description:"Name of the S3 bucket containing log files" category:"source" example:"my-cloudtrail-logs" llmguidance:"Provide just the bucket name, not the full ARN or path. Example: 'my-logs-bucket'"` + + AccessKey string `json:"access_key" yaml:"access_key" description:"AWS access key ID for S3 authentication" category:"auth" sensitive:"true" llmguidance:"AWS access key typically starts with 'AKIA'. Requires IAM permissions: s3:ListBucket, s3:GetObject, and s3:DeleteObject (unless single_load is true)"` + + SecretKey string `json:"secret_key,omitempty" yaml:"secret_key,omitempty" description:"AWS secret access key corresponding to the access key" category:"auth" sensitive:"true" llmguidance:"Keep this value secret. Never log or expose in plain text"` + + IsOneTimeLoad bool `json:"single_load" yaml:"single_load" description:"If true, loads all files once without deletion. If false, continuously monitors bucket and deletes files after successful ingestion" category:"behavior" default:"false" llmguidance:"Set to true for one-time historical data imports. Set to false for continuous live monitoring. When false, files are deleted after processing to prevent re-ingestion"` + + Prefix string `json:"prefix" yaml:"prefix" description:"S3 key prefix to filter which objects to process" category:"source" example:"AWSLogs/123456789012/CloudTrail/" llmguidance:"Optional. Use to limit processing to specific paths within the bucket. Leave empty to process all objects in the bucket"` + + ParallelFetch int `json:"parallel_fetch" yaml:"parallel_fetch" description:"Number of parallel file downloads from S3" category:"performance" default:"1" llmguidance:"Increase for better throughput on large buckets. Recommended: 5-10 for most use cases. Higher values use more memory"` + + Region string `json:"region" yaml:"region" description:"AWS region where the bucket is located" category:"source" example:"us-east-1" llmguidance:"Optional. If not specified, the region will be auto-detected. Specify to avoid the auto-detection API call"` +} + +func (c *S3Config) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.BucketName == "" { + return errors.New("missing bucket_name") + } + if c.AccessKey == "" { + return errors.New("missing access_key") + } + if c.SecretKey == "" { + return errors.New("missing secret_key") + } + return nil +} diff --git a/adaptertypes/sentinelone.go b/adaptertypes/sentinelone.go new file mode 100644 index 0000000..d14f04b --- /dev/null +++ b/adaptertypes/sentinelone.go @@ -0,0 +1,40 @@ +package adaptertypes + +import ( + "errors" + "fmt" + "strings" + "time" +) + +type SentinelOneConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Domain string `json:"domain" yaml:"domain" description:"SentinelOne console domain" category:"source" example:"mycompany.sentinelone.net" llmguidance:"Your SentinelOne console domain (without https://)"` + APIKey string `json:"api_key" yaml:"api_key" description:"SentinelOne API key" category:"auth" sensitive:"true" llmguidance:"Generate in SentinelOne console under Settings > Users > Service Users"` + URLs string `json:"urls" yaml:"urls" description:"Comma-separated list of SentinelOne API endpoints to query" category:"source" example:"/web/api/v2.1/activities" llmguidance:"API endpoints to poll for data. Common: /web/api/v2.1/activities, /web/api/v2.1/threats"` + StartTime string `json:"start_time" yaml:"start_time" description:"Start time for data collection" category:"behavior" example:"2024-01-01T00:00:00Z" llmguidance:"ISO 8601 format timestamp. Events before this time are ignored"` + TimeBetweenRequests time.Duration `json:"time_between_requests" yaml:"time_between_requests" description:"Duration to wait between API requests" category:"performance" default:"60s" llmguidance:"Go duration format (e.g., '60s', '5m'). Prevents API rate limiting"` +} + +func (c *SentinelOneConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Domain == "" { + return errors.New("missing domain") + } + if c.APIKey == "" { + return errors.New("missing api_key") + } + if !strings.HasPrefix(c.Domain, "https://") { + c.Domain = "https://" + c.Domain + } + c.Domain = strings.TrimSuffix(c.Domain, "/") + if _, err := time.Parse("2006-01-02T15:04:05.999999Z", c.StartTime); c.StartTime != "" && err != nil { + return fmt.Errorf("invalid start_time: %v", err) + } + if c.TimeBetweenRequests == 0 { + c.TimeBetweenRequests = 1 * time.Minute + } + return nil +} diff --git a/adaptertypes/shared.go b/adaptertypes/shared.go new file mode 100644 index 0000000..3fcdb62 --- /dev/null +++ b/adaptertypes/shared.go @@ -0,0 +1,21 @@ +package adaptertypes + +import ( + "io" + + "github.com/refractionPOINT/go-uspclient" + "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/utils" +) + +// Re-export common types used across all adapters +// This keeps the adaptertypes package self-contained while avoiding duplication + +type ClientOptions = uspclient.ClientOptions +type Identity = uspclient.Identity +type AckBufferOptions = uspclient.AckBufferOptions +type MappingDescriptor = protocol.MappingDescriptor +type IndexDescriptor = protocol.IndexDescriptor +type FieldMapping = protocol.FieldMapping +type Deduper = utils.Deduper +type ReadCloser = io.ReadCloser diff --git a/adaptertypes/simulator.go b/adaptertypes/simulator.go new file mode 100644 index 0000000..0e7740d --- /dev/null +++ b/adaptertypes/simulator.go @@ -0,0 +1,18 @@ +package adaptertypes + +import "fmt" + +// SimulatorConfig defines the configuration for the simulator adapter (testing/replay) +type SimulatorConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Reader ReadCloser `json:"-" yaml:"-"` + FilePath string `json:"file_path" yaml:"file_path" description:"Path to file containing events to replay" category:"source"` + IsReplayTiming bool `json:"is_replay_timing" yaml:"is_replay_timing" description:"Replay events with original timing" category:"behavior" default:"false" llmguidance:"If true, respects timestamp gaps between events. If false, sends as fast as possible"` +} + +func (c *SimulatorConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + return nil +} diff --git a/adaptertypes/slack.go b/adaptertypes/slack.go new file mode 100644 index 0000000..09cc894 --- /dev/null +++ b/adaptertypes/slack.go @@ -0,0 +1,22 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// SlackConfig defines the configuration for the Slack audit logs adapter +type SlackConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Token string `json:"token" yaml:"token" description:"Slack API token for audit log access" category:"auth" sensitive:"true" llmguidance:"Requires an Enterprise Grid organization and audit logs API access"` +} + +func (c *SlackConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Token == "" { + return errors.New("missing token") + } + return nil +} diff --git a/adaptertypes/sophos.go b/adaptertypes/sophos.go new file mode 100644 index 0000000..d36d443 --- /dev/null +++ b/adaptertypes/sophos.go @@ -0,0 +1,34 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// SophosConfig defines the configuration for the Sophos Central adapter +type SophosConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ClientId string `json:"clientid" yaml:"clientid" description:"Sophos Central API client ID" category:"auth"` + ClientSecret string `json:"clientsecret" yaml:"clientsecret" description:"Sophos Central API client secret" category:"auth" sensitive:"true"` + TenantId string `json:"tenantid" yaml:"tenantid" description:"Sophos Central tenant ID" category:"auth"` + URL string `json:"url" yaml:"url" description:"Sophos Central API URL" category:"source" example:"https://api-us01.central.sophos.com"` +} + +func (c *SophosConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.URL == "" { + return errors.New("missing url") + } + if c.ClientId == "" { + return errors.New("missing client id") + } + if c.ClientSecret == "" { + return errors.New("missing client secret") + } + if c.TenantId == "" { + return errors.New("missing tenant id") + } + return nil +} diff --git a/adaptertypes/sqs.go b/adaptertypes/sqs.go new file mode 100644 index 0000000..682bac9 --- /dev/null +++ b/adaptertypes/sqs.go @@ -0,0 +1,34 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// SQSConfig defines the configuration for the AWS SQS adapter +type SQSConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + AccessKey string `json:"access_key" yaml:"access_key" description:"AWS access key ID" category:"auth" sensitive:"true"` + SecretKey string `json:"secret_key,omitempty" yaml:"secret_key,omitempty" description:"AWS secret access key" category:"auth" sensitive:"true"` + QueueURL string `json:"queue_url" yaml:"queue_url" description:"SQS queue URL" category:"source" example:"https://sqs.us-east-1.amazonaws.com/123456789012/my-queue"` + Region string `json:"region" yaml:"region" description:"AWS region" category:"source" example:"us-east-1"` +} + +func (c *SQSConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.AccessKey == "" { + return errors.New("missing access_key") + } + if c.SecretKey == "" { + return errors.New("missing secret_key") + } + if c.Region == "" { + return errors.New("missing region") + } + if c.QueueURL == "" { + return errors.New("missing queue_url") + } + return nil +} diff --git a/adaptertypes/sqs_files.go b/adaptertypes/sqs_files.go new file mode 100644 index 0000000..19ed088 --- /dev/null +++ b/adaptertypes/sqs_files.go @@ -0,0 +1,39 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// SQSFilesConfig defines the configuration for the AWS SQS+S3 files adapter +type SQSFilesConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + AccessKey string `json:"access_key" yaml:"access_key" description:"AWS access key ID" category:"auth" sensitive:"true"` + SecretKey string `json:"secret_key,omitempty" yaml:"secret_key,omitempty" description:"AWS secret access key" category:"auth" sensitive:"true"` + QueueURL string `json:"queue_url" yaml:"queue_url" description:"SQS queue URL receiving S3 notifications" category:"source" example:"https://sqs.us-east-1.amazonaws.com/123456789012/my-queue"` + Region string `json:"region" yaml:"region" description:"AWS region" category:"source" example:"us-east-1"` + ParallelFetch int `json:"parallel_fetch" yaml:"parallel_fetch" description:"Number of parallel file downloads" category:"performance" default:"1"` + BucketPath string `json:"bucket_path,omitempty" yaml:"bucket_path,omitempty" description:"S3 bucket path from SQS message" category:"parsing" llmguidance:"JSON path to bucket name in SQS message"` + FilePath string `json:"file_path,omitempty" yaml:"file_path,omitempty" description:"S3 file key path from SQS message" category:"parsing" llmguidance:"JSON path to object key in SQS message"` + IsDecodeObjectKey bool `json:"is_decode_object_key,omitempty" yaml:"is_decode_object_key,omitempty" description:"URL decode the object key from SQS message" category:"parsing" default:"false"` + Bucket string `json:"bucket,omitempty" yaml:"bucket,omitempty" description:"Override bucket name from SQS message" category:"source" llmguidance:"Optional. Use if all files are in same bucket"` +} + +func (c *SQSFilesConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.AccessKey == "" { + return errors.New("missing access_key") + } + if c.SecretKey == "" { + return errors.New("missing secret_key") + } + if c.Region == "" { + return errors.New("missing region") + } + if c.QueueURL == "" { + return errors.New("missing queue_url") + } + return nil +} diff --git a/adaptertypes/stdin.go b/adaptertypes/stdin.go new file mode 100644 index 0000000..8643b98 --- /dev/null +++ b/adaptertypes/stdin.go @@ -0,0 +1,16 @@ +package adaptertypes + +import "fmt" + +// StdinConfig defines the configuration for the stdin adapter +type StdinConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` +} + +func (c *StdinConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + return nil +} diff --git a/adaptertypes/sublime.go b/adaptertypes/sublime.go new file mode 100644 index 0000000..b33e1f3 --- /dev/null +++ b/adaptertypes/sublime.go @@ -0,0 +1,28 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +const defaultBaseURL = "https://platform.sublime.security" + +// SublimeConfig defines the configuration for the Sublime Security adapter +type SublimeConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ApiKey string `json:"api_key" yaml:"api_key" description:"Sublime Security API key" category:"auth" sensitive:"true"` + BaseURL string `json:"base_url" yaml:"base_url" description:"Sublime Security API base URL" category:"source" example:"https://api.sublime.security"` +} + +func (c *SublimeConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ApiKey == "" { + return errors.New("missing api key") + } + if c.BaseURL == "" { + c.BaseURL = defaultBaseURL + } + return nil +} diff --git a/adaptertypes/syslog.go b/adaptertypes/syslog.go new file mode 100644 index 0000000..2727924 --- /dev/null +++ b/adaptertypes/syslog.go @@ -0,0 +1,27 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +type SyslogConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + Port uint16 `json:"port" yaml:"port" description:"Port number to listen on for syslog messages" category:"source" example:"514" llmguidance:"Standard syslog port is 514 (requires root). Use 1514 or higher for non-root"` + Interface string `json:"iface" yaml:"iface" description:"Network interface to bind to" category:"source" example:"0.0.0.0" llmguidance:"Use 0.0.0.0 to listen on all interfaces, or specify a specific IP address"` + IsUDP bool `json:"is_udp,omitempty" yaml:"is_udp,omitempty" description:"Use UDP instead of TCP" category:"behavior" default:"false" llmguidance:"UDP is stateless and faster but less reliable. TCP is recommended for critical logs"` + SslCertPath string `json:"ssl_cert" yaml:"ssl_cert" description:"Path to SSL/TLS certificate file" category:"auth" example:"/etc/ssl/certs/server.crt" llmguidance:"Required for TLS syslog. PEM format certificate file"` + SslKeyPath string `json:"ssl_key" yaml:"ssl_key" description:"Path to SSL/TLS private key file" category:"auth" sensitive:"true" example:"/etc/ssl/private/server.key" llmguidance:"Required for TLS syslog. PEM format private key file"` + MutualTlsCertPath string `json:"mutual_tls_cert,omitempty" yaml:"mutual_tls_cert,omitempty" description:"Path to mutual TLS CA certificate for client authentication" category:"auth" example:"/etc/ssl/certs/ca.crt" llmguidance:"Optional. Enable mutual TLS by providing CA cert to verify client certificates"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` +} + +func (c *SyslogConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.Port == 0 { + return errors.New("missing port") + } + return nil +} diff --git a/adaptertypes/trendmicro.go b/adaptertypes/trendmicro.go new file mode 100644 index 0000000..a669810 --- /dev/null +++ b/adaptertypes/trendmicro.go @@ -0,0 +1,39 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +var regionalDomains = map[string]string{ + "us": "https://api.xdr.trendmicro.com", + "eu": "https://api.eu.xdr.trendmicro.com", + "sg": "https://api.sg.xdr.trendmicro.com", + "jp": "https://api.xdr.trendmicro.co.jp", + "in": "https://api.in.xdr.trendmicro.com", + "au": "https://api.au.xdr.trendmicro.com", +} + +// TrendMicroConfig defines the configuration for the Trend Micro adapter +type TrendMicroConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + APIToken string `json:"api_token" yaml:"api_token" description:"Trend Micro API token" category:"auth" sensitive:"true"` + Region string `json:"region" yaml:"region" description:"Trend Micro region" category:"source" example:"us" llmguidance:"Options: 'us', 'eu', 'sg', 'au', 'in', 'jp'"` +} + +func (c *TrendMicroConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + + if c.APIToken == "" { + return errors.New("missing api_token") + } + if c.Region == "" { + c.Region = "us" + } + if _, ok := regionalDomains[c.Region]; !ok { + return fmt.Errorf("invalid region: %s (must be one of: us, eu, sg, jp, in, au)", c.Region) + } + return nil +} diff --git a/adaptertypes/types.go b/adaptertypes/types.go new file mode 100644 index 0000000..4047831 --- /dev/null +++ b/adaptertypes/types.go @@ -0,0 +1,66 @@ +package adaptertypes + +// AdapterConfigs is the central registry of all adapter configuration types. +// This struct is used for reflection-based schema generation. +type AdapterConfigs struct { + // Cloud Storage + S3 S3Config `json:"s3" yaml:"s3" adapterName:"s3" adapterCategory:"cloud_storage" description:"Amazon S3 bucket log ingestion"` + GCS GCSConfig `json:"gcs" yaml:"gcs" adapterName:"gcs" adapterCategory:"cloud_storage" description:"Google Cloud Storage bucket log ingestion"` + + // File-Based + File FileConfig `json:"file" yaml:"file" adapterName:"file" adapterCategory:"file_based" description:"Local file and directory monitoring"` + EVTX EVTXConfig `json:"evtx" yaml:"evtx" adapterName:"evtx" adapterCategory:"file_based" description:"Windows Event Log (.evtx) file parser"` + WEL WELConfig `json:"wel" yaml:"wel" adapterName:"wel" adapterCategory:"file_based" description:"Live Windows Event Log monitoring"` + Syslog SyslogConfig `json:"syslog" yaml:"syslog" adapterName:"syslog" adapterCategory:"file_based" description:"Syslog server (UDP/TCP/TLS)"` + Stdin StdinConfig `json:"stdin" yaml:"stdin" adapterName:"stdin" adapterCategory:"file_based" description:"Standard input stream ingestion"` + Simulator SimulatorConfig `json:"simulator" yaml:"simulator" adapterName:"simulator" adapterCategory:"file_based" description:"Event replay simulator for testing"` + + // Authentication & IAM + OnePassword OnePasswordConfig `json:"1password" yaml:"1password" adapterName:"1password" adapterCategory:"authentication" description:"1Password event logs"` + Bitwarden BitwardenConfig `json:"bitwarden" yaml:"bitwarden" adapterName:"bitwarden" adapterCategory:"authentication" description:"Bitwarden event logs"` + Duo DuoConfig `json:"duo" yaml:"duo" adapterName:"duo" adapterCategory:"authentication" description:"Duo Security authentication logs"` + Okta OktaConfig `json:"okta" yaml:"okta" adapterName:"okta" adapterCategory:"authentication" description:"Okta system logs"` + EntraID EntraIDConfig `json:"entraid" yaml:"entraid" adapterName:"entraid" adapterCategory:"authentication" description:"Microsoft Entra ID (Azure AD) audit logs"` + + // Security Products + Defender DefenderConfig `json:"defender" yaml:"defender" adapterName:"defender" adapterCategory:"security_products" description:"Microsoft Defender for Endpoint"` + SentinelOne SentinelOneConfig `json:"sentinel_one" yaml:"sentinel_one" adapterName:"sentinel_one" adapterCategory:"security_products" description:"SentinelOne EDR platform"` + FalconCloud FalconCloudConfig `json:"falconcloud" yaml:"falconcloud" adapterName:"falconcloud" adapterCategory:"security_products" description:"CrowdStrike Falcon"` + Cylance CylanceConfig `json:"cylance" yaml:"cylance" adapterName:"cylance" adapterCategory:"security_products" description:"BlackBerry Cylance"` + Sophos SophosConfig `json:"sophos" yaml:"sophos" adapterName:"sophos" adapterCategory:"security_products" description:"Sophos Central"` + TrendMicro TrendMicroConfig `json:"trendmicro" yaml:"trendmicro" adapterName:"trendmicro" adapterCategory:"security_products" description:"Trend Micro security platform"` + Wiz WizConfig `json:"wiz" yaml:"wiz" adapterName:"wiz" adapterCategory:"security_products" description:"Wiz cloud security platform"` + ProofpointTap ProofpointTapConfig `json:"proofpoint_tap" yaml:"proofpoint_tap" adapterName:"proofpoint_tap" adapterCategory:"security_products" description:"Proofpoint Targeted Attack Protection"` + Sublime SublimeConfig `json:"sublime" yaml:"sublime" adapterName:"sublime" adapterCategory:"security_products" description:"Sublime Security email security"` + + // Cloud Platforms + PubSub PubSubConfig `json:"pubsub" yaml:"pubsub" adapterName:"pubsub" adapterCategory:"cloud_platforms" description:"Google Cloud Pub/Sub"` + BigQuery BigQueryConfig `json:"bigquery" yaml:"bigquery" adapterName:"bigquery" adapterCategory:"cloud_platforms" description:"Google BigQuery data warehouse"` + AzureEventHub EventHubConfig `json:"azure_event_hub" yaml:"azure_event_hub" adapterName:"azure_event_hub" adapterCategory:"cloud_platforms" description:"Azure Event Hubs"` + SQS SQSConfig `json:"sqs" yaml:"sqs" adapterName:"sqs" adapterCategory:"cloud_platforms" description:"AWS Simple Queue Service"` + SQSFiles SQSFilesConfig `json:"sqs-files" yaml:"sqs-files" adapterName:"sqs-files" adapterCategory:"cloud_platforms" description:"AWS SQS with S3 file downloads"` + K8sPods K8sPodsConfig `json:"k8s_pods" yaml:"k8s_pods" adapterName:"k8s_pods" adapterCategory:"cloud_platforms" description:"Kubernetes pod logs"` + MacUnifiedLogging MacUnifiedLoggingConfig `json:"mac_unified_logging" yaml:"mac_unified_logging" adapterName:"mac_unified_logging" adapterCategory:"cloud_platforms" description:"macOS Unified Logging System"` + + // Communication & Collaboration + Slack SlackConfig `json:"slack" yaml:"slack" adapterName:"slack" adapterCategory:"communication" description:"Slack audit logs"` + Office365 Office365Config `json:"office365" yaml:"office365" adapterName:"office365" adapterCategory:"communication" description:"Office 365 Management Activity API"` + MsGraph MsGraphConfig `json:"ms_graph" yaml:"ms_graph" adapterName:"ms_graph" adapterCategory:"communication" description:"Microsoft Graph API"` + Mimecast MimecastConfig `json:"mimecast" yaml:"mimecast" adapterName:"mimecast" adapterCategory:"communication" description:"Mimecast email security"` + IMAP ImapConfig `json:"imap" yaml:"imap" adapterName:"imap" adapterCategory:"communication" description:"IMAP email monitoring"` + + // Business Tools + HubSpot HubSpotConfig `json:"hubspot" yaml:"hubspot" adapterName:"hubspot" adapterCategory:"business_tools" description:"HubSpot CRM"` + ITGlue ITGlueConfig `json:"itglue" yaml:"itglue" adapterName:"itglue" adapterCategory:"business_tools" description:"IT Glue documentation platform"` + Zendesk ZendeskConfig `json:"zendesk" yaml:"zendesk" adapterName:"zendesk" adapterCategory:"business_tools" description:"Zendesk customer support platform"` + PandaDoc PandaDocConfig `json:"pandadoc" yaml:"pandadoc" adapterName:"pandadoc" adapterCategory:"business_tools" description:"PandaDoc document management"` + Box BoxConfig `json:"box" yaml:"box" adapterName:"box" adapterCategory:"business_tools" description:"Box cloud storage and collaboration"` + + // Network + Cato CatoConfig `json:"cato" yaml:"cato" adapterName:"cato" adapterCategory:"network" description:"Cato Networks SASE platform"` +} + +// GetAdapterRegistry returns a new instance of the adapter registry +func GetAdapterRegistry() *AdapterConfigs { + return &AdapterConfigs{} +} diff --git a/adaptertypes/wel.go b/adaptertypes/wel.go new file mode 100644 index 0000000..0b3a9eb --- /dev/null +++ b/adaptertypes/wel.go @@ -0,0 +1,23 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// WELConfig defines the configuration for the Windows Event Log adapter +type WELConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + EvtSources string `json:"evt_sources,omitempty" yaml:"evt_sources,omitempty" description:"Comma-separated list of Windows Event Log sources to monitor" category:"source" example:"Application,Security,System" llmguidance:"Common sources: Application, Security, System, Microsoft-Windows-Sysmon/Operational"` + WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"` +} + +func (c *WELConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.EvtSources == "" { + return errors.New("missing evt_sources, a csv of SOURCE-NAME:FILTER") + } + return nil +} diff --git a/adaptertypes/wiz.go b/adaptertypes/wiz.go new file mode 100644 index 0000000..aa5a16e --- /dev/null +++ b/adaptertypes/wiz.go @@ -0,0 +1,47 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// WizConfig defines the configuration for the Wiz security platform adapter +type WizConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ClientID string `json:"client_id" yaml:"client_id" description:"Wiz API client ID" category:"auth"` + ClientSecret string `json:"client_secret" yaml:"client_secret" description:"Wiz API client secret" category:"auth" sensitive:"true"` + URL string `json:"url" yaml:"url" description:"Wiz API URL" category:"source" example:"https://api.us1.app.wiz.io/graphql"` + Query string `json:"query" yaml:"query" description:"GraphQL query to execute" category:"source" llmguidance:"Custom GraphQL query for fetching data"` + Variables map[string]interface{} `json:"variables" yaml:"variables" description:"Variables for the GraphQL query" category:"source"` + TimeField string `json:"time_field" yaml:"time_field" description:"Field name containing timestamp" category:"parsing" example:"createdAt"` + DataPath []string `json:"data_path" yaml:"data_path" description:"Path to data in GraphQL response" category:"parsing" llmguidance:"Array of keys to navigate to the data array in response"` + IDField string `json:"id_field" yaml:"id_field" description:"Field name containing unique ID for deduplication" category:"parsing" example:"id"` +} + +func (c *WizConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ClientID == "" { + return errors.New("missing client_id") + } + if c.ClientSecret == "" { + return errors.New("missing client_secret") + } + if c.URL == "" { + return errors.New("missing url") + } + if c.Query == "" { + return errors.New("missing query") + } + if c.TimeField == "" { + return errors.New("missing time_field") + } + if len(c.DataPath) == 0 { + return errors.New("missing data_path") + } + if c.IDField == "" { + return errors.New("missing id_field") + } + return nil +} diff --git a/adaptertypes/zendesk.go b/adaptertypes/zendesk.go new file mode 100644 index 0000000..380f888 --- /dev/null +++ b/adaptertypes/zendesk.go @@ -0,0 +1,31 @@ +package adaptertypes + +import ( + "errors" + "fmt" +) + +// ZendeskConfig defines the configuration for the Zendesk adapter +type ZendeskConfig struct { + ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"` + ApiToken string `json:"api_token" yaml:"api_token" description:"Zendesk API token" category:"auth" sensitive:"true"` + ZendeskDomain string `json:"zendesk_domain" yaml:"zendesk_domain" description:"Zendesk domain" category:"source" example:"mycompany.zendesk.com" llmguidance:"Your Zendesk subdomain"` + ZendeskEmail string `json:"zendesk_email" yaml:"zendesk_email" description:"Zendesk admin email address" category:"auth" llmguidance:"Email of the admin user who owns the API token"` +} + +func (c *ZendeskConfig) Validate() error { + if err := c.ClientOptions.Validate(); err != nil { + return fmt.Errorf("client_options: %v", err) + } + if c.ApiToken == "" { + return errors.New("missing api token") + } + if c.ZendeskDomain == "" { + return errors.New("missing zendesk domain (e.g., 'your-company.zendesk.com')") + } + if c.ZendeskEmail == "" { + return errors.New("missing zendesk email") + } + + return nil +} diff --git a/azure_event_hub/client.go b/azure_event_hub/client.go index 4b37eed..c4b3ba6 100644 --- a/azure_event_hub/client.go +++ b/azure_event_hub/client.go @@ -6,10 +6,11 @@ import ( "fmt" "time" - "github.com/Azure/azure-event-hubs-go/v3" + eventhub "github.com/Azure/azure-event-hubs-go/v3" "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) const ( @@ -17,7 +18,7 @@ const ( ) type EventHubAdapter struct { - conf EventHubConfig + conf adaptertypes.EventHubConfig uspClient *uspclient.Client hub *eventhub.Hub @@ -27,22 +28,7 @@ type EventHubAdapter struct { chStopped chan struct{} } -type EventHubConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ConnectionString string `json:"connection_string" yaml:"connection_string"` -} - -func (c *EventHubConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ConnectionString == "" { - return errors.New("missing connection_string") - } - return nil -} - -func NewEventHubAdapter(conf EventHubConfig) (*EventHubAdapter, chan struct{}, error) { +func NewEventHubAdapter(conf adaptertypes.EventHubConfig) (*EventHubAdapter, chan struct{}, error) { a := &EventHubAdapter{ conf: conf, ctx: context.Background(), diff --git a/bigquery/client.go b/bigquery/client.go index bc50313..c382279 100644 --- a/bigquery/client.go +++ b/bigquery/client.go @@ -7,8 +7,6 @@ import ( "context" "errors" "fmt" - "github.com/refractionPOINT/go-uspclient" - "github.com/refractionPOINT/go-uspclient/protocol" "strings" "sync" "time" @@ -16,10 +14,14 @@ import ( "cloud.google.com/go/bigquery" "google.golang.org/api/iterator" "google.golang.org/api/option" + + "github.com/refractionPOINT/go-uspclient" + "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) type BigQueryAdapter struct { - conf BigQueryConfig + conf adaptertypes.BigQueryConfig client *bigquery.Client dataset *bigquery.Dataset table *bigquery.Table @@ -30,27 +32,7 @@ type BigQueryAdapter struct { cancel context.CancelFunc } -func (bq *BigQueryConfig) Validate() error { - if bq.ProjectId == "" { - return errors.New("missing project_id") - } - // this will usually be th same as projectID but could be different if using outside project dataset such as a public data set - if bq.BigQueryProject == "" { - return errors.New("missing bigquery project name") - } - if bq.DatasetName == "" { - return errors.New("missing dataset_name") - } - if bq.TableName == "" { - return errors.New("missing table_name") - } - if bq.SqlQuery == "" { - return errors.New("missing sql query") - } - return nil -} - -func NewBigQueryAdapter(conf BigQueryConfig) (*BigQueryAdapter, chan struct{}, error) { +func NewBigQueryAdapter(conf adaptertypes.BigQueryConfig) (*BigQueryAdapter, chan struct{}, error) { // Create bq cancellable context ctx, cancel := context.WithCancel(context.Background()) bq := &BigQueryAdapter{ diff --git a/bitwarden/client.go b/bitwarden/client.go index fcb2f08..e2f563b 100644 --- a/bitwarden/client.go +++ b/bitwarden/client.go @@ -15,6 +15,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -25,56 +26,8 @@ const ( eventsBaseURLEU = "https://api.bitwarden.eu" ) -type BitwardenConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` - Region string `json:"region" yaml:"region"` // "us" or "eu", defaults to "us" - TokenEndpointURL string `json:"token_endpoint_url" yaml:"token_endpoint_url"` // Custom token endpoint URL for self-hosted instances - EventsBaseURL string `json:"events_base_url" yaml:"events_base_url"` // Custom events base URL for self-hosted instances -} - -func (c *BitwardenConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - - if c.ClientID == "" { - return errors.New("missing client_id") - } - if c.ClientSecret == "" { - return errors.New("missing client_secret") - } - - // Check if custom URLs are provided - hasCustomURLs := c.TokenEndpointURL != "" || c.EventsBaseURL != "" - - if hasCustomURLs { - // If custom URLs are provided, both must be set and region must be empty - if c.TokenEndpointURL == "" { - return errors.New("token_endpoint_url must be set when events_base_url is provided") - } - if c.EventsBaseURL == "" { - return errors.New("events_base_url must be set when token_endpoint_url is provided") - } - if c.Region != "" { - return errors.New("region cannot be set when using custom URLs (token_endpoint_url and events_base_url)") - } - } else { - // If custom URLs are not provided, use region-based configuration - if c.Region == "" { - c.Region = "us" - } - if c.Region != "us" && c.Region != "eu" { - return fmt.Errorf("invalid region: %s (must be 'us' or 'eu')", c.Region) - } - } - - return nil -} - type BitwardenAdapter struct { - conf BitwardenConfig + conf adaptertypes.BitwardenConfig uspClient *uspclient.Client httpClient *http.Client chStopped chan struct{} @@ -89,7 +42,7 @@ type BitwardenAdapter struct { lastStartDate *time.Time } -func NewBitwardenAdapter(conf BitwardenConfig) (*BitwardenAdapter, chan struct{}, error) { +func NewBitwardenAdapter(conf adaptertypes.BitwardenConfig) (*BitwardenAdapter, chan struct{}, error) { if err := conf.Validate(); err != nil { return nil, nil, err } diff --git a/box/client.go b/box/client.go index 68a6684..97b69d2 100644 --- a/box/client.go +++ b/box/client.go @@ -15,6 +15,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -23,26 +24,8 @@ const ( tokenEndpoint = "https://api.box.com/oauth2/token" ) -type BoxConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` - SubjectID string `json:"subject_id" yaml:"subject_id"` -} - -func (c *BoxConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - - if c.ClientID == "" || c.ClientSecret == "" || c.SubjectID == "" { - return errors.New("missing Box client ID, secret, or subject ID") - } - return nil -} - type BoxAdapter struct { - conf BoxConfig + conf adaptertypes.BoxConfig uspClient *uspclient.Client httpClient *http.Client chStopped chan struct{} @@ -54,7 +37,7 @@ type BoxAdapter struct { initialized bool } -func NewBoxAdapter(conf BoxConfig) (*BoxAdapter, chan struct{}, error) { +func NewBoxAdapter(conf adaptertypes.BoxConfig) (*BoxAdapter, chan struct{}, error) { var err error a := &BoxAdapter{ conf: conf, diff --git a/cato/client.go b/cato/client.go index 3bd222d..b821eb9 100644 --- a/cato/client.go +++ b/cato/client.go @@ -5,7 +5,6 @@ import ( "compress/gzip" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "log" @@ -18,6 +17,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "golang.org/x/net/context/ctxhttp" ) @@ -34,28 +34,8 @@ var ( configFile string = "./config.txt" ) -type CatoConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` - ApiKey string `json:"apikey" yaml:"apikey"` - AccountId int `json:"accountid" yaml:"accountid"` -} - -func (c *CatoConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.AccountId == 0 { - return errors.New("missing account id") - } - if c.ApiKey == "" { - return errors.New("missing api key") - } - return nil -} - type CatoAdapter struct { - conf CatoConfig + conf adaptertypes.CatoConfig wg sync.WaitGroup isRunning uint32 mRunning sync.RWMutex @@ -68,7 +48,7 @@ type CatoAdapter struct { ctx context.Context } -func NewCatoAdapter(conf CatoConfig) (*CatoAdapter, chan struct{}, error) { +func NewCatoAdapter(conf adaptertypes.CatoConfig) (*CatoAdapter, chan struct{}, error) { a := &CatoAdapter{ conf: conf, isRunning: 1, diff --git a/containers/conf/all.go b/containers/conf/all.go index 55784b9..23d5e2f 100755 --- a/containers/conf/all.go +++ b/containers/conf/all.go @@ -1,90 +1,50 @@ package conf import ( - usp_bigquery "github.com/refractionPOINT/usp-adapters/bigquery" - - "github.com/refractionPOINT/usp-adapters/1password" - "github.com/refractionPOINT/usp-adapters/azure_event_hub" - "github.com/refractionPOINT/usp-adapters/bitwarden" - "github.com/refractionPOINT/usp-adapters/box" - "github.com/refractionPOINT/usp-adapters/cato" - "github.com/refractionPOINT/usp-adapters/cylance" - "github.com/refractionPOINT/usp-adapters/defender" - "github.com/refractionPOINT/usp-adapters/duo" - "github.com/refractionPOINT/usp-adapters/entraid" - "github.com/refractionPOINT/usp-adapters/evtx" - "github.com/refractionPOINT/usp-adapters/falconcloud" - "github.com/refractionPOINT/usp-adapters/file" - "github.com/refractionPOINT/usp-adapters/gcs" - "github.com/refractionPOINT/usp-adapters/hubspot" - "github.com/refractionPOINT/usp-adapters/imap" - "github.com/refractionPOINT/usp-adapters/itglue" - "github.com/refractionPOINT/usp-adapters/k8s_pods" - "github.com/refractionPOINT/usp-adapters/mac_unified_logging" - "github.com/refractionPOINT/usp-adapters/mimecast" - "github.com/refractionPOINT/usp-adapters/ms_graph" - "github.com/refractionPOINT/usp-adapters/o365" - "github.com/refractionPOINT/usp-adapters/okta" - "github.com/refractionPOINT/usp-adapters/pandadoc" - "github.com/refractionPOINT/usp-adapters/proofpoint_tap" - "github.com/refractionPOINT/usp-adapters/pubsub" - "github.com/refractionPOINT/usp-adapters/s3" - "github.com/refractionPOINT/usp-adapters/sentinelone" - "github.com/refractionPOINT/usp-adapters/simulator" - "github.com/refractionPOINT/usp-adapters/slack" - "github.com/refractionPOINT/usp-adapters/sophos" - "github.com/refractionPOINT/usp-adapters/sqs" - "github.com/refractionPOINT/usp-adapters/sqs-files" - "github.com/refractionPOINT/usp-adapters/stdin" - "github.com/refractionPOINT/usp-adapters/sublime" - "github.com/refractionPOINT/usp-adapters/syslog" - "github.com/refractionPOINT/usp-adapters/trendmicro" - "github.com/refractionPOINT/usp-adapters/wel" - "github.com/refractionPOINT/usp-adapters/wiz" - "github.com/refractionPOINT/usp-adapters/zendesk" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) type GeneralConfigs struct { Healthcheck int `json:"healthcheck" yaml:"healthcheck"` - Syslog usp_syslog.SyslogConfig `json:"syslog" yaml:"syslog"` - PubSub usp_pubsub.PubSubConfig `json:"pubsub" yaml:"pubsub"` - S3 usp_s3.S3Config `json:"s3" yaml:"s3"` - Stdin usp_stdin.StdinConfig `json:"stdin" yaml:"stdin"` - OnePassword usp_1password.OnePasswordConfig `json:"1password" yaml:"1password"` - Bitwarden usp_bitwarden.BitwardenConfig `json:"bitwarden" yaml:"bitwarden"` - ITGlue usp_itglue.ITGlueConfig `json:"itglue" yaml:"itglue"` - Sophos usp_sophos.SophosConfig `json:"sophos" yaml:"sophos"` - EntraID usp_entraid.EntraIDConfig `json:"entraid" yaml:"entraid"` - Defender usp_defender.DefenderConfig `json:"defender" yaml:"defender"` - Cato usp_cato.CatoConfig `json:"cato" yaml:"cato"` - Cylance usp_cylance.CylanceConfig `json:"cylance" yaml:"cylance"` - Okta usp_okta.OktaConfig `json:"okta" yaml:"okta"` - Office365 usp_o365.Office365Config `json:"office365" yaml:"office365"` - Wel usp_wel.WELConfig `json:"wel" yaml:"wel"` - MacUnifiedLogging usp_mac_unified_logging.MacUnifiedLoggingConfig `json:"mac_unified_logging" yaml:"mac_unified_logging"` - AzureEventHub usp_azure_event_hub.EventHubConfig `json:"azure_event_hub" yaml:"azure_event_hub"` - Duo usp_duo.DuoConfig `json:"duo" yaml:"duo"` - Gcs usp_gcs.GCSConfig `json:"gcs" yaml:"gcs"` - Slack usp_slack.SlackConfig `json:"slack" yaml:"slack"` - Sqs usp_sqs.SQSConfig `json:"sqs" yaml:"sqs"` - SqsFiles usp_sqs_files.SQSFilesConfig `json:"sqs-files" yaml:"sqs-files"` - Simulator usp_simulator.SimulatorConfig `json:"simulator" yaml:"simulator"` - File usp_file.FileConfig `json:"file" yaml:"file"` - Evtx usp_evtx.EVTXConfig `json:"evtx" yaml:"evtx"` - K8sPods usp_k8s_pods.K8sPodsConfig `json:"k8s_pods" yaml:"k8s_pods"` - BigQuery usp_bigquery.BigQueryConfig `json:"bigquery" yaml:"bigquery"` - Imap usp_imap.ImapConfig `json:"imap" yaml:"imap"` - HubSpot usp_hubspot.HubSpotConfig `json:"hubspot" yaml:"hubspot"` - FalconCloud usp_falconcloud.FalconCloudConfig `json:"falconcloud" yaml:"falconcloud"` - Mimecast usp_mimecast.MimecastConfig `json:"mimecast" yaml:"mimecast"` - MsGraph usp_ms_graph.MsGraphConfig `json:"ms_graph" yaml:"ms_graph"` - Zendesk usp_zendesk.ZendeskConfig `json:"zendesk" yaml:"zendesk"` - PandaDoc usp_pandadoc.PandaDocConfig `json:"pandadoc" yaml:"pandadoc"` - ProofpointTap usp_proofpoint_tap.ProofpointTapConfig `json:"proofpoint_tap" yaml:"proofpoint_tap"` - Box usp_box.BoxConfig `json:"box" yaml:"box"` - Sublime usp_sublime.SublimeConfig `json:"sublime" yaml:"sublime"` - SentinelOne usp_sentinelone.SentinelOneConfig `json:"sentinel_one" yaml:"sentinel_one"` - TrendMicro usp_trendmicro.TrendMicroConfig `json:"trendmicro" yaml:"trendmicro"` - Wiz usp_wiz.WizConfig `json:"wiz" yaml:"wiz"` + Syslog adaptertypes.SyslogConfig `json:"syslog" yaml:"syslog"` + PubSub adaptertypes.PubSubConfig `json:"pubsub" yaml:"pubsub"` + S3 adaptertypes.S3Config `json:"s3" yaml:"s3"` + Stdin adaptertypes.StdinConfig `json:"stdin" yaml:"stdin"` + OnePassword adaptertypes.OnePasswordConfig `json:"1password" yaml:"1password"` + Bitwarden adaptertypes.BitwardenConfig `json:"bitwarden" yaml:"bitwarden"` + ITGlue adaptertypes.ITGlueConfig `json:"itglue" yaml:"itglue"` + Sophos adaptertypes.SophosConfig `json:"sophos" yaml:"sophos"` + EntraID adaptertypes.EntraIDConfig `json:"entraid" yaml:"entraid"` + Defender adaptertypes.DefenderConfig `json:"defender" yaml:"defender"` + Cato adaptertypes.CatoConfig `json:"cato" yaml:"cato"` + Cylance adaptertypes.CylanceConfig `json:"cylance" yaml:"cylance"` + Okta adaptertypes.OktaConfig `json:"okta" yaml:"okta"` + Office365 adaptertypes.Office365Config `json:"office365" yaml:"office365"` + Wel adaptertypes.WELConfig `json:"wel" yaml:"wel"` + MacUnifiedLogging adaptertypes.MacUnifiedLoggingConfig `json:"mac_unified_logging" yaml:"mac_unified_logging"` + AzureEventHub adaptertypes.EventHubConfig `json:"azure_event_hub" yaml:"azure_event_hub"` + Duo adaptertypes.DuoConfig `json:"duo" yaml:"duo"` + Gcs adaptertypes.GCSConfig `json:"gcs" yaml:"gcs"` + Slack adaptertypes.SlackConfig `json:"slack" yaml:"slack"` + Sqs adaptertypes.SQSConfig `json:"sqs" yaml:"sqs"` + SqsFiles adaptertypes.SQSFilesConfig `json:"sqs-files" yaml:"sqs-files"` + Simulator adaptertypes.SimulatorConfig `json:"simulator" yaml:"simulator"` + File adaptertypes.FileConfig `json:"file" yaml:"file"` + Evtx adaptertypes.EVTXConfig `json:"evtx" yaml:"evtx"` + K8sPods adaptertypes.K8sPodsConfig `json:"k8s_pods" yaml:"k8s_pods"` + BigQuery adaptertypes.BigQueryConfig `json:"bigquery" yaml:"bigquery"` + Imap adaptertypes.ImapConfig `json:"imap" yaml:"imap"` + HubSpot adaptertypes.HubSpotConfig `json:"hubspot" yaml:"hubspot"` + FalconCloud adaptertypes.FalconCloudConfig `json:"falconcloud" yaml:"falconcloud"` + Mimecast adaptertypes.MimecastConfig `json:"mimecast" yaml:"mimecast"` + MsGraph adaptertypes.MsGraphConfig `json:"ms_graph" yaml:"ms_graph"` + Zendesk adaptertypes.ZendeskConfig `json:"zendesk" yaml:"zendesk"` + PandaDoc adaptertypes.PandaDocConfig `json:"pandadoc" yaml:"pandadoc"` + ProofpointTap adaptertypes.ProofpointTapConfig `json:"proofpoint_tap" yaml:"proofpoint_tap"` + Box adaptertypes.BoxConfig `json:"box" yaml:"box"` + Sublime adaptertypes.SublimeConfig `json:"sublime" yaml:"sublime"` + SentinelOne adaptertypes.SentinelOneConfig `json:"sentinel_one" yaml:"sentinel_one"` + TrendMicro adaptertypes.TrendMicroConfig `json:"trendmicro" yaml:"trendmicro"` + Wiz adaptertypes.WizConfig `json:"wiz" yaml:"wiz"` } diff --git a/cylance/client.go b/cylance/client.go index b30de64..83ba158 100644 --- a/cylance/client.go +++ b/cylance/client.go @@ -16,6 +16,7 @@ import ( "github.com/google/uuid" "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -30,16 +31,8 @@ const ( memoryProtectionEndpoint = "/memoryprotection/v2" ) -type CylanceConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - TenantID string `json:"tenant_id" yaml:"tenant_id"` - AppID string `json:"app_id" yaml:"app_id"` - AppSecret string `json:"app_secret" yaml:"app_secret"` - LoggingBaseURL string `json:"logging_base_url" yaml:"logging_base_url"` -} - type CylanceAdapter struct { - conf CylanceConfig + conf adaptertypes.CylanceConfig uspClient *uspclient.Client httpClient *http.Client chStopped chan struct{} @@ -57,7 +50,7 @@ type CylanceAdapter struct { refreshFailLimitMet bool } -func NewCylanceAdapter(conf CylanceConfig) (*CylanceAdapter, chan struct{}, error) { +func NewCylanceAdapter(conf adaptertypes.CylanceConfig) (*CylanceAdapter, chan struct{}, error) { if err := conf.Validate(); err != nil { return nil, nil, err } @@ -98,25 +91,6 @@ func NewCylanceAdapter(conf CylanceConfig) (*CylanceAdapter, chan struct{}, erro return a, a.chStopped, nil } -func (c *CylanceConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.TenantID == "" { - return errors.New("missing tenant id") - } - if c.AppID == "" { - return errors.New("missing app id") - } - if c.AppSecret == "" { - return errors.New("missing app secret") - } - if c.LoggingBaseURL == "" { - c.LoggingBaseURL = defaultLoggingBaseURL - } - return nil -} - func (a *CylanceAdapter) Close() error { a.conf.ClientOptions.DebugLog("closing") var err1, err2 error diff --git a/defender/client.go b/defender/client.go index 13da601..c9627c2 100644 --- a/defender/client.go +++ b/defender/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -15,6 +14,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -24,7 +24,7 @@ var URL = map[string]string{ } type DefenderAdapter struct { - conf DefenderConfig + conf adaptertypes.DefenderConfig uspClient *uspclient.Client httpClient *http.Client @@ -37,30 +37,7 @@ type DefenderAdapter struct { ctx context.Context } -type DefenderConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - TenantID string `json:"tenant_id" yaml:"tenant_id"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` -} - -func (c *DefenderConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.TenantID == "" { - return errors.New("missing tenant_id") - } - if c.ClientID == "" { - return errors.New("missing client_id") - } - if c.ClientSecret == "" { - return errors.New("missing client_secret") - } - return nil -} - -func NewDefenderAdapter(conf DefenderConfig) (*DefenderAdapter, chan struct{}, error) { +func NewDefenderAdapter(conf adaptertypes.DefenderConfig) (*DefenderAdapter, chan struct{}, error) { var err error a := &DefenderAdapter{ conf: conf, diff --git a/duo/client.go b/duo/client.go index 617af74..b8027b3 100644 --- a/duo/client.go +++ b/duo/client.go @@ -2,13 +2,13 @@ package usp_duo import ( "context" - "errors" "fmt" "sync" "time" "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" "github.com/duosecurity/duo_api_golang" @@ -20,7 +20,7 @@ const ( ) type DuoAdapter struct { - conf DuoConfig + conf adaptertypes.DuoConfig uspClient *uspclient.Client duoClient *duoapi.DuoApi adminClient *duoadmin.Client @@ -32,30 +32,7 @@ type DuoAdapter struct { ctx context.Context } -type DuoConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - IntegrationKey string `json:"integration_key" yaml:"integration_key"` - SecretKey string `json:"secret_key" yaml:"secret_key"` - APIHostname string `json:"api_hostname" yaml:"api_hostname"` -} - -func (c *DuoConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.IntegrationKey == "" { - return errors.New("missing integration_key") - } - if c.SecretKey == "" { - return errors.New("missing secret_key") - } - if c.APIHostname == "" { - return errors.New("missing api_hostname") - } - return nil -} - -func NewDuoAdapter(conf DuoConfig) (*DuoAdapter, chan struct{}, error) { +func NewDuoAdapter(conf adaptertypes.DuoConfig) (*DuoAdapter, chan struct{}, error) { var err error a := &DuoAdapter{ conf: conf, diff --git a/entraid/client.go b/entraid/client.go index 256e118..6c10fff 100644 --- a/entraid/client.go +++ b/entraid/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -15,6 +14,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -24,7 +24,7 @@ var URL = map[string]string{ } type EntraIDAdapter struct { - conf EntraIDConfig + conf adaptertypes.EntraIDConfig uspClient *uspclient.Client httpClient *http.Client @@ -37,30 +37,7 @@ type EntraIDAdapter struct { ctx context.Context } -type EntraIDConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - TenantID string `json:"tenant_id" yaml:"tenant_id"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` -} - -func (c *EntraIDConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.TenantID == "" { - return errors.New("missing tenant_id") - } - if c.ClientID == "" { - return errors.New("missing client_id") - } - if c.ClientSecret == "" { - return errors.New("missing client_secret") - } - return nil -} - -func NewEntraIDAdapter(conf EntraIDConfig) (*EntraIDAdapter, chan struct{}, error) { +func NewEntraIDAdapter(conf adaptertypes.EntraIDConfig) (*EntraIDAdapter, chan struct{}, error) { var err error a := &EntraIDAdapter{ conf: conf, diff --git a/evtx/client.go b/evtx/client.go index 9f0c33e..f478a93 100644 --- a/evtx/client.go +++ b/evtx/client.go @@ -2,7 +2,6 @@ package usp_evtx import ( "encoding/json" - "errors" "fmt" "os" "sync" @@ -10,6 +9,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/evtx" ) @@ -19,7 +19,7 @@ const ( ) type EVTXAdapter struct { - conf EVTXConfig + conf adaptertypes.EVTXConfig wg sync.WaitGroup uspClient *uspclient.Client writeTimeout time.Duration @@ -28,23 +28,7 @@ type EVTXAdapter struct { fClose func() } -type EVTXConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` - FilePath string `json:"file_path" yaml:"file_path"` -} - -func (c *EVTXConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.FilePath == "" { - return errors.New("file_path missing") - } - return nil -} - -func NewEVTXAdapter(conf EVTXConfig) (*EVTXAdapter, chan struct{}, error) { +func NewEVTXAdapter(conf adaptertypes.EVTXConfig) (*EVTXAdapter, chan struct{}, error) { a := &EVTXAdapter{ conf: conf, } diff --git a/falconcloud/client.go b/falconcloud/client.go index 961e7b8..019c7ab 100644 --- a/falconcloud/client.go +++ b/falconcloud/client.go @@ -3,7 +3,6 @@ package usp_falconcloud import ( "context" "encoding/json" - "errors" "fmt" "sync" "time" @@ -11,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/crowdstrike/gofalcon/falcon" "github.com/crowdstrike/gofalcon/falcon/client" @@ -22,31 +22,8 @@ const ( defaultWriteTimeout = 60 * 10 ) -type FalconCloudConfig struct { - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ClientId string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` - IsUsingOffset bool `json:"is_using_offset" yaml:"is_using_offset"` - Offset uint64 `json:"offset" yaml:"offset"` - NotBefore *time.Time `json:"not_before,omitempty" yaml:"not_before,omitempty"` -} - -func (c *FalconCloudConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ClientId == "" { - return errors.New("missing client id") - } - if c.ClientSecret == "" { - return errors.New("missing client secret") - } - return nil -} - type FalconCloudAdapter struct { - conf FalconCloudConfig + conf adaptertypes.FalconCloudConfig isRunning uint32 mRunning sync.RWMutex uspClient *uspclient.Client @@ -59,7 +36,7 @@ type FalconCloudAdapter struct { cancel context.CancelFunc } -func NewFalconCloudAdapter(conf FalconCloudConfig) (*FalconCloudAdapter, chan struct{}, error) { +func NewFalconCloudAdapter(conf adaptertypes.FalconCloudConfig) (*FalconCloudAdapter, chan struct{}, error) { a := &FalconCloudAdapter{ conf: conf, isRunning: 1, diff --git a/file/client.go b/file/client.go index ac3ce54..a7f1915 100644 --- a/file/client.go +++ b/file/client.go @@ -5,7 +5,6 @@ package usp_file import ( "context" - "errors" "fmt" "io" "os" @@ -18,6 +17,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/nxadm/tail" @@ -58,7 +58,7 @@ type tailInfo struct { type FileAdapter struct { ctx context.Context - conf FileConfig + conf adaptertypes.FileConfig wg sync.WaitGroup uspClient *uspclient.Client writeTimeout time.Duration @@ -68,17 +68,7 @@ type FileAdapter struct { lineCb func(line string) // callback for each line for testing } -func (c *FileConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.FilePath == "" { - return errors.New("file_path missing") - } - return nil -} - -func NewFileAdapter(conf FileConfig) (*FileAdapter, chan struct{}, error) { +func NewFileAdapter(conf adaptertypes.FileConfig) (*FileAdapter, chan struct{}, error) { a := &FileAdapter{ conf: conf, tailFiles: make(map[string]*tailInfo), diff --git a/file/client_test.go b/file/client_test.go index 907a8ae..dfec292 100644 --- a/file/client_test.go +++ b/file/client_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/refractionPOINT/go-uspclient" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -42,7 +43,7 @@ func TestPollFiles(t *testing.T) { mockClientOptions := new(MockClientOptions) // Create FileAdapter instance adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 5, ReactivationThreshold: 10, @@ -131,7 +132,7 @@ func TestPollSerialFiles(t *testing.T) { } // Create FileAdapter instance adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 5, ReactivationThreshold: 10, @@ -216,7 +217,7 @@ func TestTailActiveFile(t *testing.T) { // Create FileAdapter instance adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 2, // Set high to prevent inactivity ReactivationThreshold: 1, @@ -323,7 +324,7 @@ func TestMultiLineJSON(t *testing.T) { // Create FileAdapter instance with MultiLineJSON enabled adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.json"), InactivityThreshold: 5, ReactivationThreshold: 1, @@ -440,7 +441,7 @@ func TestFileRotationDetection(t *testing.T) { require.NoError(t, err) adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 60, // High to prevent inactivity ReactivationThreshold: 10, @@ -579,7 +580,7 @@ func TestFileRotationPreservesData(t *testing.T) { require.NoError(t, err) adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 120, ReactivationThreshold: 10, @@ -689,7 +690,7 @@ func TestMultipleFileRotations(t *testing.T) { require.NoError(t, err) adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 120, Backfill: true, @@ -794,7 +795,7 @@ func TestConcurrentFileRotations(t *testing.T) { require.NoError(t, err) adapter := &FileAdapter{ - conf: FileConfig{ + conf: adaptertypes.FileConfig{ FilePath: filepath.Join(tmpDir, "*.log"), InactivityThreshold: 120, Backfill: true, diff --git a/file/conf.go b/file/conf.go deleted file mode 100644 index 6450e23..0000000 --- a/file/conf.go +++ /dev/null @@ -1,18 +0,0 @@ -package usp_file - -import ( - "github.com/refractionPOINT/go-uspclient" -) - -type FileConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` - FilePath string `json:"file_path" yaml:"file_path"` - NoFollow bool `json:"no_follow" yaml:"no_follow"` - InactivityThreshold int `json:"inactivity_threshold" yaml:"inactivity_threshold"` - ReactivationThreshold int `json:"reactivation_threshold" yaml:"reactivation_threshold"` - Backfill bool `json:"backfill" yaml:"backfill"` - SerializeFiles bool `json:"serialize_files" yaml:"serialize_files"` - Poll bool `json:"poll" yaml:"poll"` - MultiLineJSON bool `json:"multi_line_json" yaml:"multi_line_json"` -} diff --git a/gcs/client.go b/gcs/client.go index 591b173..2bb36d2 100644 --- a/gcs/client.go +++ b/gcs/client.go @@ -2,7 +2,6 @@ package usp_gcs import ( "context" - "errors" "fmt" "io" "strings" @@ -16,13 +15,14 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) const maxObjectSize = 1024 * 1024 * 100 // 100 MB type GCSAdapter struct { - conf GCSConfig + conf adaptertypes.GCSConfig uspClient *uspclient.Client ctx context.Context @@ -34,25 +34,6 @@ type GCSAdapter struct { wg sync.WaitGroup } -type GCSConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - BucketName string `json:"bucket_name" yaml:"bucket_name"` - ServiceAccountCreds string `json:"service_account_creds,omitempty" yaml:"service_account_creds,omitempty"` - IsOneTimeLoad bool `json:"single_load" yaml:"single_load"` - Prefix string `json:"prefix" yaml:"prefix"` - ParallelFetch int `json:"parallel_fetch" yaml:"parallel_fetch"` -} - -func (c *GCSConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.BucketName == "" { - return errors.New("missing bucket_name") - } - return nil -} - type gcsLocalFile struct { Obj *storage.ObjectHandle Attrs *storage.ObjectAttrs @@ -61,7 +42,7 @@ type gcsLocalFile struct { Err error } -func NewGCSAdapter(conf GCSConfig) (*GCSAdapter, chan struct{}, error) { +func NewGCSAdapter(conf adaptertypes.GCSConfig) (*GCSAdapter, chan struct{}, error) { if conf.ParallelFetch <= 0 { conf.ParallelFetch = 1 } diff --git a/go.mod b/go.mod index 2c1373d..cb9bfe4 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/refractionPOINT/gjson v0.0.0-20230509223721-3a6dd216c22d github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20250728014624-2a3d8e58cb31 github.com/refractionPOINT/go-uspclient v1.5.1 + github.com/refractionPOINT/usp-adapters/adaptertypes v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.10.0 github.com/vmihailenco/msgpack/v5 v5.4.1 golang.org/x/net v0.42.0 @@ -145,3 +146,5 @@ require ( ) replace github.com/nxadm/tail => github.com/refractionPOINT/tail v0.0.0-20211216163028-4472660a31a6 + +replace github.com/refractionPOINT/usp-adapters/adaptertypes => ./adaptertypes diff --git a/hubspot/client.go b/hubspot/client.go index 1ff0c55..8677e26 100644 --- a/hubspot/client.go +++ b/hubspot/client.go @@ -3,7 +3,6 @@ package usp_hubspot import ( "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -13,6 +12,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -28,7 +28,7 @@ type opRequest struct { } type HubSpotAdapter struct { - conf HubSpotConfig + conf adaptertypes.HubSpotConfig uspClient *uspclient.Client httpClient *http.Client @@ -41,22 +41,7 @@ type HubSpotAdapter struct { dedupe map[string]int64 } -type HubSpotConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - AccessToken string `json:"access_token" yaml:"access_token"` -} - -func (c *HubSpotConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.AccessToken == "" { - return errors.New("missing access token") - } - return nil -} - -func NewHubSpotAdapter(conf HubSpotConfig) (*HubSpotAdapter, chan struct{}, error) { +func NewHubSpotAdapter(conf adaptertypes.HubSpotConfig) (*HubSpotAdapter, chan struct{}, error) { var err error a := &HubSpotAdapter{ conf: conf, diff --git a/imap/client.go b/imap/client.go index 0c9ba81..a1a83c2 100644 --- a/imap/client.go +++ b/imap/client.go @@ -19,6 +19,7 @@ import ( "github.com/refractionPOINT/go-limacharlie/limacharlie" "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) var ( @@ -26,7 +27,7 @@ var ( ) type IMAPAdapter struct { - conf ImapConfig + conf adaptertypes.ImapConfig uspClient *uspclient.Client imapClient *client.Client @@ -39,37 +40,7 @@ type IMAPAdapter struct { ctx context.Context } -type ImapConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Server string `json:"server" yaml:"server"` - UserName string `json:"username" yaml:"username"` - Password string `json:"password" yaml:"password"` - InboxName string `json:"inbox_name" yaml:"inbox_name"` - IsInsecure bool `json:"is_insecure" yaml:"is_insecure"` - FromZero bool `json:"from_zero" yaml:"from_zero"` - IncludeAttachments bool `json:"include_attachments" yaml:"include_attachments"` - MaxBodySize int `json:"max_body_size" yaml:"max_body_size"` - AttachmentIngestKey string `json:"attachment_ingest_key" yaml:"attachment_ingest_key"` - AttachmentRetentionDays int `json:"attachment_retention_days" yaml:"attachment_retention_days"` -} - -func (c *ImapConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Server == "" { - return errors.New("missing server") - } - if c.UserName == "" { - return errors.New("missing username") - } - if c.Password == "" { - return errors.New("missing password") - } - return nil -} - -func NewImapAdapter(conf ImapConfig) (*IMAPAdapter, chan struct{}, error) { +func NewImapAdapter(conf adaptertypes.ImapConfig) (*IMAPAdapter, chan struct{}, error) { a := &IMAPAdapter{ conf: conf, ctx: context.Background(), diff --git a/itglue/client.go b/itglue/client.go index 37ef32e..191be5e 100644 --- a/itglue/client.go +++ b/itglue/client.go @@ -3,7 +3,6 @@ package usp_itglue import ( "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -13,6 +12,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -28,7 +28,7 @@ type opRequest struct { } type ITGlueAdapter struct { - conf ITGlueConfig + conf adaptertypes.ITGlueConfig uspClient *uspclient.Client httpClient *http.Client @@ -39,22 +39,7 @@ type ITGlueAdapter struct { ctx context.Context } -type ITGlueConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Token string `json:"token" yaml:"token"` -} - -func (c *ITGlueConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Token == "" { - return errors.New("missing token") - } - return nil -} - -func NewITGlueAdapter(conf ITGlueConfig) (*ITGlueAdapter, chan struct{}, error) { +func NewITGlueAdapter(conf adaptertypes.ITGlueConfig) (*ITGlueAdapter, chan struct{}, error) { var err error a := &ITGlueAdapter{ conf: conf, diff --git a/k8s_pods/client.go b/k8s_pods/client.go index d288b3b..9a93254 100644 --- a/k8s_pods/client.go +++ b/k8s_pods/client.go @@ -4,7 +4,6 @@ package usp_k8s_pods import ( - "errors" "fmt" "regexp" "sync" @@ -12,6 +11,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -20,7 +20,7 @@ const ( ) type K8sPodsAdapter struct { - conf K8sPodsConfig + conf adaptertypes.K8sPodsConfig uspClient *uspclient.Client writeTimeout time.Duration wg sync.WaitGroup @@ -35,17 +35,7 @@ type runtimeOptions struct { excludePods *regexp.Regexp } -func (c *K8sPodsConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Root == "" { - return errors.New("file_path missing") - } - return nil -} - -func NewK8sPodsAdapter(conf K8sPodsConfig) (*K8sPodsAdapter, chan struct{}, error) { +func NewK8sPodsAdapter(conf adaptertypes.K8sPodsConfig) (*K8sPodsAdapter, chan struct{}, error) { a := &K8sPodsAdapter{ conf: conf, } diff --git a/mac_unified_logging/client.go b/mac_unified_logging/client.go index 417e39d..5247367 100644 --- a/mac_unified_logging/client.go +++ b/mac_unified_logging/client.go @@ -15,6 +15,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) const ( @@ -22,7 +23,7 @@ const ( ) type MacUnifiedLoggingAdapter struct { - conf MacUnifiedLoggingConfig + conf adaptertypes.MacUnifiedLoggingConfig wg sync.WaitGroup isRunning uint32 mRunning sync.RWMutex @@ -35,7 +36,7 @@ type MacUnifiedLoggingAdapter struct { ctx context.Context } -func NewMacUnifiedLoggingAdapter(conf MacUnifiedLoggingConfig) (*MacUnifiedLoggingAdapter, chan struct{}, error) { +func NewMacUnifiedLoggingAdapter(conf adaptertypes.MacUnifiedLoggingConfig) (*MacUnifiedLoggingAdapter, chan struct{}, error) { a := &MacUnifiedLoggingAdapter{ conf: conf, isRunning: 1, diff --git a/mac_unified_logging/conf.go b/mac_unified_logging/conf.go deleted file mode 100644 index 5e7ac1b..0000000 --- a/mac_unified_logging/conf.go +++ /dev/null @@ -1,11 +0,0 @@ -package usp_mac_unified_logging - -import ( - "github.com/refractionPOINT/go-uspclient" -) - -type MacUnifiedLoggingConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` - Predicate string `json:"predicate,omitempty" yaml:"predicate,omitempty"` -} diff --git a/mac_unified_logging/noop.go b/mac_unified_logging/noop.go index 2743dc0..717a02d 100644 --- a/mac_unified_logging/noop.go +++ b/mac_unified_logging/noop.go @@ -3,7 +3,11 @@ package usp_mac_unified_logging -import "errors" +import ( + "errors" + + "github.com/refractionPOINT/usp-adapters/adaptertypes" +) // Dummy noop file to build when the platform // is _not_ MacOS since unified logging is only @@ -11,7 +15,7 @@ import "errors" type MacUnifiedLoggingAdapter struct{} -func NewMacUnifiedLoggingAdapter(conf MacUnifiedLoggingConfig) (*MacUnifiedLoggingAdapter, chan struct{}, error) { +func NewMacUnifiedLoggingAdapter(conf adaptertypes.MacUnifiedLoggingConfig) (*MacUnifiedLoggingAdapter, chan struct{}, error) { return nil, nil, errors.New("mac (MacOS unified logging) collection not supported outside of MacOS") } diff --git a/mimecast/client.go b/mimecast/client.go index 44075ae..5ed9c76 100644 --- a/mimecast/client.go +++ b/mimecast/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -14,6 +13,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -23,7 +23,7 @@ const ( ) type MimecastAdapter struct { - conf MimecastConfig + conf adaptertypes.MimecastConfig uspClient *uspclient.Client httpClient *http.Client @@ -85,27 +85,7 @@ type AuditLog struct { Category string `json:"category"` } -type MimecastConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ClientId string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` -} - -func (c *MimecastConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ClientId == "" { - return errors.New("missing client id") - } - if c.ClientSecret == "" { - return errors.New("missing client secret") - } - - return nil -} - -func NewMimecastAdapter(conf MimecastConfig) (*MimecastAdapter, chan struct{}, error) { +func NewMimecastAdapter(conf adaptertypes.MimecastConfig) (*MimecastAdapter, chan struct{}, error) { var err error a := &MimecastAdapter{ conf: conf, diff --git a/ms_graph/client.go b/ms_graph/client.go index 201ba48..caa102f 100644 --- a/ms_graph/client.go +++ b/ms_graph/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -15,6 +14,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -22,7 +22,7 @@ const scope = "https://graph.microsoft.com/.default" const URLPrefix = "https://graph.microsoft.com/v1.0/" type MsGraphAdapter struct { - conf MsGraphConfig + conf adaptertypes.MsGraphConfig uspClient *uspclient.Client httpClient *http.Client @@ -35,34 +35,7 @@ type MsGraphAdapter struct { ctx context.Context } -type MsGraphConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - TenantID string `json:"tenant_id" yaml:"tenant_id"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` - URL string `json:"url" yaml:"url"` -} - -func (c *MsGraphConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.TenantID == "" { - return errors.New("missing tenant_id") - } - if c.ClientID == "" { - return errors.New("missing client_id") - } - if c.ClientSecret == "" { - return errors.New("missing client_secret") - } - if c.URL == "" { - return errors.New("missing url") - } - return nil -} - -func NewMsGraphAdapter(conf MsGraphConfig) (*MsGraphAdapter, chan struct{}, error) { +func NewMsGraphAdapter(conf adaptertypes.MsGraphConfig) (*MsGraphAdapter, chan struct{}, error) { var err error a := &MsGraphAdapter{ conf: conf, diff --git a/o365/client.go b/o365/client.go index 713009d..ac31c9e 100644 --- a/o365/client.go +++ b/o365/client.go @@ -17,6 +17,7 @@ import ( "github.com/refractionPOINT/gjson" "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" "golang.org/x/oauth2/clientcredentials" @@ -61,7 +62,7 @@ var ResourceScope = map[string]string{ } type Office365Adapter struct { - conf Office365Config + conf adaptertypes.Office365Config uspClient *uspclient.Client httpClient *http.Client @@ -75,50 +76,7 @@ type Office365Adapter struct { ctx context.Context } -type Office365Config struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Domain string `json:"domain" yaml:"domain"` - TenantID string `json:"tenant_id" yaml:"tenant_id"` - PublisherID string `json:"publisher_id" yaml:"publisher_id"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` - Endpoint string `json:"endpoint" yaml:"endpoint"` - ContentTypes string `json:"content_types" yaml:"content_types"` - StartTime string `json:"start_time" yaml:"start_time"` - - Deduper utils.Deduper `json:"-" yaml:"-"` -} - -func (c *Office365Config) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Domain == "" { - return errors.New("missing domain") - } - if c.TenantID == "" { - return errors.New("missing tenant_id") - } - if c.PublisherID == "" { - return errors.New("missing publisher_id") - } - if c.ClientID == "" { - return errors.New("missing client_id") - } - if c.ClientSecret == "" { - return errors.New("missing client_secret") - } - if c.Endpoint == "" { - return errors.New("missing endpoint") - } - _, ok := URL[c.Endpoint] - if !strings.HasPrefix(c.Endpoint, "https://") && !ok { - return fmt.Errorf("invalid endpoint, not https or in %v", URL) - } - return nil -} - -func NewOffice365Adapter(conf Office365Config) (*Office365Adapter, chan struct{}, error) { +func NewOffice365Adapter(conf adaptertypes.Office365Config) (*Office365Adapter, chan struct{}, error) { var err error // If no deduper is provided, use a local one. diff --git a/okta/client.go b/okta/client.go index 47da776..0dd31c6 100644 --- a/okta/client.go +++ b/okta/client.go @@ -3,7 +3,6 @@ package usp_okta import ( "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -13,6 +12,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -28,7 +28,7 @@ type opRequest struct { } type OktaAdapter struct { - conf OktaConfig + conf adaptertypes.OktaConfig uspClient *uspclient.Client httpClient *http.Client @@ -41,26 +41,7 @@ type OktaAdapter struct { dedupe map[string]int64 } -type OktaConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ApiKey string `json:"apikey" yaml:"apikey"` - URL string `json:"url" yaml:"url"` -} - -func (c *OktaConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.URL == "" { - return errors.New("missing url") - } - if c.ApiKey == "" { - return errors.New("missing api key") - } - return nil -} - -func NewOktaAdapter(conf OktaConfig) (*OktaAdapter, chan struct{}, error) { +func NewOktaAdapter(conf adaptertypes.OktaConfig) (*OktaAdapter, chan struct{}, error) { var err error a := &OktaAdapter{ conf: conf, diff --git a/pandadoc/client.go b/pandadoc/client.go index 62129d9..57d6190 100644 --- a/pandadoc/client.go +++ b/pandadoc/client.go @@ -3,7 +3,6 @@ package usp_pandadoc import ( "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -13,6 +12,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -28,7 +28,7 @@ type opRequest struct { } type PandaDocAdapter struct { - conf PandaDocConfig + conf adaptertypes.PandaDocConfig uspClient *uspclient.Client httpClient *http.Client @@ -41,23 +41,7 @@ type PandaDocAdapter struct { dedupe map[string]int64 } -type PandaDocConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ApiKey string `json:"api_key" yaml:"api_key"` -} - -func (c *PandaDocConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ApiKey == "" { - return errors.New("missing api key") - } - - return nil -} - -func NewPandaDocAdapter(conf PandaDocConfig) (*PandaDocAdapter, chan struct{}, error) { +func NewPandaDocAdapter(conf adaptertypes.PandaDocConfig) (*PandaDocAdapter, chan struct{}, error) { var err error a := &PandaDocAdapter{ conf: conf, diff --git a/proofpoint_tap/client.go b/proofpoint_tap/client.go index 779d2c0..8be07dc 100644 --- a/proofpoint_tap/client.go +++ b/proofpoint_tap/client.go @@ -2,7 +2,6 @@ package usp_proofpoint_tap import ( "encoding/json" - "errors" "fmt" "io" "net" @@ -12,6 +11,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -22,7 +22,7 @@ const ( ) type ProofpointTapAdapter struct { - conf ProofpointTapConfig + conf adaptertypes.ProofpointTapConfig uspClient *uspclient.Client httpClient *http.Client @@ -34,26 +34,7 @@ type ProofpointTapAdapter struct { clickDedupe map[string]int64 } -type ProofpointTapConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Principal string `json:"principal" yaml:"principal"` - Secret string `json:"secret" yaml:"secret"` -} - -func (c *ProofpointTapConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Principal == "" { - return errors.New("missing principal") - } - if c.Secret == "" { - return errors.New("missing secret") - } - return nil -} - -func NewProofpointTapAdapter(conf ProofpointTapConfig) (*ProofpointTapAdapter, chan struct{}, error) { +func NewProofpointTapAdapter(conf adaptertypes.ProofpointTapConfig) (*ProofpointTapAdapter, chan struct{}, error) { if err := conf.Validate(); err != nil { return nil, nil, err } diff --git a/pubsub/client.go b/pubsub/client.go index efad7c6..4c53723 100644 --- a/pubsub/client.go +++ b/pubsub/client.go @@ -2,7 +2,6 @@ package usp_pubsub import ( "context" - "errors" "fmt" "strings" "time" @@ -12,6 +11,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) const ( @@ -19,7 +19,7 @@ const ( ) type PubSubAdapter struct { - conf PubSubConfig + conf adaptertypes.PubSubConfig uspClient *uspclient.Client psClient *pubsub.Client @@ -29,31 +29,7 @@ type PubSubAdapter struct { stopSub context.CancelFunc } -type PubSubConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - SubscriptionName string `json:"sub_name" yaml:"sub_name"` - ProjectName string `json:"project_name" yaml:"project_name"` - ServiceAccountCreds string `json:"service_account_creds,omitempty" yaml:"service_account_creds,omitempty"` - MaxPSBuffer int `json:"max_ps_buffer,omitempty" yaml:"max_ps_buffer,omitempty"` -} - -func (c *PubSubConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.SubscriptionName == "" { - return errors.New("missing sub_name") - } - if c.ProjectName == "" { - return errors.New("missing project_name") - } - if c.ServiceAccountCreds == "" { - return errors.New("missing service_account_creds") - } - return nil -} - -func NewPubSubAdapter(conf PubSubConfig) (*PubSubAdapter, chan struct{}, error) { +func NewPubSubAdapter(conf adaptertypes.PubSubConfig) (*PubSubAdapter, chan struct{}, error) { a := &PubSubAdapter{ conf: conf, ctx: context.Background(), diff --git a/s3/client.go b/s3/client.go index 217a5c5..345ed2b 100644 --- a/s3/client.go +++ b/s3/client.go @@ -2,7 +2,6 @@ package usp_s3 import ( "context" - "errors" "fmt" "net/http" "strings" @@ -21,6 +20,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -59,7 +59,7 @@ func (r connResetRetryer) ShouldRetry(req *request.Request) bool { const maxObjectSize = 1024 * 1024 * 100 // 100 MB type S3Adapter struct { - conf S3Config + conf adaptertypes.S3Config uspClient *uspclient.Client ctx context.Context @@ -75,33 +75,6 @@ type S3Adapter struct { region string } -type S3Config struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - BucketName string `json:"bucket_name" yaml:"bucket_name"` - AccessKey string `json:"access_key" yaml:"access_key"` - SecretKey string `json:"secret_key,omitempty" yaml:"secret_key,omitempty"` - IsOneTimeLoad bool `json:"single_load" yaml:"single_load"` - Prefix string `json:"prefix" yaml:"prefix"` - ParallelFetch int `json:"parallel_fetch" yaml:"parallel_fetch"` - Region string `json:"region" yaml:"region"` -} - -func (c *S3Config) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.BucketName == "" { - return errors.New("missing bucket_name") - } - if c.AccessKey == "" { - return errors.New("missing access_key") - } - if c.SecretKey == "" { - return errors.New("missing secret_key") - } - return nil -} - type s3LocalFile struct { Obj *s3Record Data []byte @@ -114,7 +87,7 @@ type s3Record struct { Size int64 } -func NewS3Adapter(conf S3Config) (*S3Adapter, chan struct{}, error) { +func NewS3Adapter(conf adaptertypes.S3Config) (*S3Adapter, chan struct{}, error) { if conf.ParallelFetch <= 0 { conf.ParallelFetch = 1 } diff --git a/sentinelone/s1.go b/sentinelone/s1.go index e072cb0..31ba639 100644 --- a/sentinelone/s1.go +++ b/sentinelone/s1.go @@ -13,11 +13,12 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) type SentinelOneAdapter struct { - conf SentinelOneConfig + conf adaptertypes.SentinelOneConfig uspClient *uspclient.Client httpClient *http.Client s1Client *SentinelOneClient @@ -30,39 +31,7 @@ type SentinelOneAdapter struct { ctx context.Context } -type SentinelOneConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Domain string `json:"domain" yaml:"domain"` - APIKey string `json:"api_key" yaml:"api_key"` - URLs string `json:"urls" yaml:"urls"` - StartTime string `json:"start_time" yaml:"start_time"` - TimeBetweenRequests time.Duration `json:"time_between_requests" yaml:"time_between_requests"` -} - -func (c *SentinelOneConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Domain == "" { - return errors.New("missing domain") - } - if c.APIKey == "" { - return errors.New("missing api_key") - } - if !strings.HasPrefix(c.Domain, "https://") { - c.Domain = "https://" + c.Domain - } - c.Domain = strings.TrimSuffix(c.Domain, "/") - if _, err := time.Parse("2006-01-02T15:04:05.999999Z", c.StartTime); c.StartTime != "" && err != nil { - return fmt.Errorf("invalid start_time: %v", err) - } - if c.TimeBetweenRequests == 0 { - c.TimeBetweenRequests = 1 * time.Minute - } - return nil -} - -func NewSentinelOneAdapter(conf SentinelOneConfig) (*SentinelOneAdapter, chan struct{}, error) { +func NewSentinelOneAdapter(conf adaptertypes.SentinelOneConfig) (*SentinelOneAdapter, chan struct{}, error) { var err error a := &SentinelOneAdapter{ diff --git a/simulator/client.go b/simulator/client.go index 4b2eda6..7d4872e 100644 --- a/simulator/client.go +++ b/simulator/client.go @@ -11,6 +11,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -20,7 +21,7 @@ const ( ) type SimulatorAdapter struct { - conf SimulatorConfig + conf adaptertypes.SimulatorConfig wg sync.WaitGroup isRunning uint32 uspClient *uspclient.Client @@ -31,27 +32,13 @@ type SimulatorAdapter struct { lastSentTime int64 } -type SimulatorConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Reader io.ReadCloser `json:"-" yaml:"-"` - FilePath string `json:"file_path" yaml:"file_path"` - IsReplayTiming bool `json:"is_replay_timing" yaml:"is_replay_timing"` -} - type basicLCEvent struct { Routing struct { EventTime int64 `json:"event_time"` } `json:"routing"` } -func (c *SimulatorConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - return nil -} - -func NewSimulatorAdapter(conf SimulatorConfig) (*SimulatorAdapter, chan struct{}, error) { +func NewSimulatorAdapter(conf adaptertypes.SimulatorConfig) (*SimulatorAdapter, chan struct{}, error) { a := &SimulatorAdapter{ conf: conf, isRunning: 1, diff --git a/slack/client.go b/slack/client.go index dcf2120..46e3af5 100644 --- a/slack/client.go +++ b/slack/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -14,6 +13,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -22,7 +22,7 @@ const ( ) type SlackAdapter struct { - conf SlackConfig + conf adaptertypes.SlackConfig uspClient *uspclient.Client httpClient *http.Client @@ -40,22 +40,7 @@ type slackResponse struct { } `json:"response_metadata"` } -type SlackConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Token string `json:"token" yaml:"token"` -} - -func (c *SlackConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Token == "" { - return errors.New("missing token") - } - return nil -} - -func NewSlackAdapter(conf SlackConfig) (*SlackAdapter, chan struct{}, error) { +func NewSlackAdapter(conf adaptertypes.SlackConfig) (*SlackAdapter, chan struct{}, error) { var err error a := &SlackAdapter{ conf: conf, diff --git a/sophos/client.go b/sophos/client.go index eb7256c..8846b40 100644 --- a/sophos/client.go +++ b/sophos/client.go @@ -3,7 +3,6 @@ package usp_sophos import ( "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -15,6 +14,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -29,7 +29,7 @@ type opRequest struct { } type SophosAdapter struct { - conf SophosConfig + conf adaptertypes.SophosConfig uspClient *uspclient.Client httpClient *http.Client @@ -40,34 +40,7 @@ type SophosAdapter struct { ctx context.Context } -type SophosConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ClientId string `json:"clientid" yaml:"clientid"` - ClientSecret string `json:"clientsecret" yaml:"clientsecret"` - TenantId string `json:"tenantid" yaml:"tenantid"` - URL string `json:"url" yaml:"url"` -} - -func (c *SophosConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.URL == "" { - return errors.New("missing url") - } - if c.ClientId == "" { - return errors.New("missing client id") - } - if c.ClientSecret == "" { - return errors.New("missing client secret") - } - if c.TenantId == "" { - return errors.New("missing tenant id") - } - return nil -} - -func NewSophosAdapter(conf SophosConfig) (*SophosAdapter, chan struct{}, error) { +func NewSophosAdapter(conf adaptertypes.SophosConfig) (*SophosAdapter, chan struct{}, error) { var err error a := &SophosAdapter{ conf: conf, diff --git a/sqs-files/client.go b/sqs-files/client.go index 1adde72..98a7700 100644 --- a/sqs-files/client.go +++ b/sqs-files/client.go @@ -19,6 +19,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -27,7 +28,7 @@ const ( ) type SQSFilesAdapter struct { - conf SQSFilesConfig + conf adaptertypes.SQSFilesConfig uspClient *uspclient.Client chFiles chan fileInfo @@ -49,49 +50,12 @@ type SQSFilesAdapter struct { wg sync.WaitGroup } -type SQSFilesConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - - // SQS specific - AccessKey string `json:"access_key" yaml:"access_key"` - SecretKey string `json:"secret_key,omitempty" yaml:"secret_key,omitempty"` - QueueURL string `json:"queue_url" yaml:"queue_url"` - Region string `json:"region" yaml:"region"` - - // S3 specific - ParallelFetch int `json:"parallel_fetch" yaml:"parallel_fetch"` - BucketPath string `json:"bucket_path,omitempty" yaml:"bucket_path,omitempty"` - FilePath string `json:"file_path,omitempty" yaml:"file_path,omitempty"` - IsDecodeObjectKey bool `json:"is_decode_object_key,omitempty" yaml:"is_decode_object_key,omitempty"` - // Optional: alternative to BucketPath - Bucket string `json:"bucket,omitempty" yaml:"bucket,omitempty"` -} - type fileInfo struct { bucket string path string } -func (c *SQSFilesConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.AccessKey == "" { - return errors.New("missing access_key") - } - if c.SecretKey == "" { - return errors.New("missing secret_key") - } - if c.Region == "" { - return errors.New("missing region") - } - if c.QueueURL == "" { - return errors.New("missing queue_url") - } - return nil -} - -func NewSQSFilesAdapter(conf SQSFilesConfig) (*SQSFilesAdapter, chan struct{}, error) { +func NewSQSFilesAdapter(conf adaptertypes.SQSFilesConfig) (*SQSFilesAdapter, chan struct{}, error) { if conf.ParallelFetch <= 0 { conf.ParallelFetch = 1 } diff --git a/sqs/client.go b/sqs/client.go index 177397f..a2c9333 100644 --- a/sqs/client.go +++ b/sqs/client.go @@ -13,6 +13,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) const ( @@ -20,7 +21,7 @@ const ( ) type SQSAdapter struct { - conf SQSConfig + conf adaptertypes.SQSConfig uspClient *uspclient.Client awsConfig *aws.Config @@ -31,34 +32,7 @@ type SQSAdapter struct { isStop bool } -type SQSConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - AccessKey string `json:"access_key" yaml:"access_key"` - SecretKey string `json:"secret_key,omitempty" yaml:"secret_key,omitempty"` - QueueURL string `json:"queue_url" yaml:"queue_url"` - Region string `json:"region" yaml:"region"` -} - -func (c *SQSConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.AccessKey == "" { - return errors.New("missing access_key") - } - if c.SecretKey == "" { - return errors.New("missing secret_key") - } - if c.Region == "" { - return errors.New("missing region") - } - if c.QueueURL == "" { - return errors.New("missing queue_url") - } - return nil -} - -func NewSQSAdapter(conf SQSConfig) (*SQSAdapter, chan struct{}, error) { +func NewSQSAdapter(conf adaptertypes.SQSConfig) (*SQSAdapter, chan struct{}, error) { a := &SQSAdapter{ conf: conf, ctx: context.Background(), diff --git a/stdin/client.go b/stdin/client.go index 6017762..74e8372 100644 --- a/stdin/client.go +++ b/stdin/client.go @@ -10,6 +10,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -18,26 +19,14 @@ const ( ) type StdinAdapter struct { - conf StdinConfig + conf adaptertypes.StdinConfig wg sync.WaitGroup isRunning uint32 uspClient *uspclient.Client writeTimeout time.Duration } -type StdinConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` -} - -func (c *StdinConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - return nil -} - -func NewStdinAdapter(conf StdinConfig) (*StdinAdapter, chan struct{}, error) { +func NewStdinAdapter(conf adaptertypes.StdinConfig) (*StdinAdapter, chan struct{}, error) { a := &StdinAdapter{ conf: conf, isRunning: 1, diff --git a/sublime/client.go b/sublime/client.go index 3f093a1..fbccc0a 100755 --- a/sublime/client.go +++ b/sublime/client.go @@ -3,7 +3,6 @@ package usp_sublime import ( "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -13,6 +12,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -24,7 +24,7 @@ const ( ) type SublimeAdapter struct { - conf SublimeConfig + conf adaptertypes.SublimeConfig uspClient *uspclient.Client httpClient *http.Client @@ -36,26 +36,7 @@ type SublimeAdapter struct { dedupe map[string]int64 } -type SublimeConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ApiKey string `json:"api_key" yaml:"api_key"` - BaseURL string `json:"base_url" yaml:"base_url"` -} - -func (c *SublimeConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ApiKey == "" { - return errors.New("missing api key") - } - if c.BaseURL == "" { - c.BaseURL = defaultBaseURL - } - return nil -} - -func NewSublimeAdapter(conf SublimeConfig) (*SublimeAdapter, chan struct{}, error) { +func NewSublimeAdapter(conf adaptertypes.SublimeConfig) (*SublimeAdapter, chan struct{}, error) { var err error a := &SublimeAdapter{ conf: conf, diff --git a/syslog/client.go b/syslog/client.go index f8d4695..61d4076 100644 --- a/syslog/client.go +++ b/syslog/client.go @@ -14,6 +14,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -23,7 +24,7 @@ const ( ) type SyslogAdapter struct { - conf SyslogConfig + conf adaptertypes.SyslogConfig listener net.Listener udpListener *net.UDPConn connMutex sync.Mutex @@ -33,28 +34,7 @@ type SyslogAdapter struct { writeTimeout time.Duration } -type SyslogConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - Port uint16 `json:"port" yaml:"port"` - Interface string `json:"iface" yaml:"iface"` - IsUDP bool `json:"is_udp,omitempty" yaml:"is_udp,omitempty"` - SslCertPath string `json:"ssl_cert" yaml:"ssl_cert"` - SslKeyPath string `json:"ssl_key" yaml:"ssl_key"` - MutualTlsCertPath string `json:"mutual_tls_cert,omitempty" yaml:"mutual_tls_cert,omitempty"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` -} - -func (c *SyslogConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.Port == 0 { - return errors.New("missing port") - } - return nil -} - -func NewSyslogAdapter(conf SyslogConfig) (*SyslogAdapter, chan struct{}, error) { +func NewSyslogAdapter(conf adaptertypes.SyslogConfig) (*SyslogAdapter, chan struct{}, error) { a := &SyslogAdapter{ conf: conf, isRunning: 1, diff --git a/trendmicro/client.go b/trendmicro/client.go index 65c7bdc..2ddd42e 100644 --- a/trendmicro/client.go +++ b/trendmicro/client.go @@ -3,7 +3,6 @@ package usp_trendmicro import ( "context" "encoding/json" - "errors" "fmt" "io" "net" @@ -14,6 +13,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -26,31 +26,8 @@ var regionalDomains = map[string]string{ "au": "https://api.au.xdr.trendmicro.com", } -type TrendMicroConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - APIToken string `json:"api_token" yaml:"api_token"` - Region string `json:"region" yaml:"region"` // "us", "eu", "sg", "jp", "in", "au" - defaults to "us" -} - -func (c *TrendMicroConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - - if c.APIToken == "" { - return errors.New("missing api_token") - } - if c.Region == "" { - c.Region = "us" - } - if _, ok := regionalDomains[c.Region]; !ok { - return fmt.Errorf("invalid region: %s (must be one of: us, eu, sg, jp, in, au)", c.Region) - } - return nil -} - type TrendMicroAdapter struct { - conf TrendMicroConfig + conf adaptertypes.TrendMicroConfig uspClient *uspclient.Client httpClient *http.Client chStopped chan struct{} @@ -61,7 +38,7 @@ type TrendMicroAdapter struct { lastFetch time.Time } -func NewTrendMicroAdapter(conf TrendMicroConfig) (*TrendMicroAdapter, chan struct{}, error) { +func NewTrendMicroAdapter(conf adaptertypes.TrendMicroConfig) (*TrendMicroAdapter, chan struct{}, error) { if err := conf.Validate(); err != nil { return nil, nil, err } diff --git a/wel/client.go b/wel/client.go index 5647fa9..843cfa1 100644 --- a/wel/client.go +++ b/wel/client.go @@ -12,6 +12,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" ) const ( @@ -19,7 +20,7 @@ const ( ) type WELAdapter struct { - conf WELConfig + conf adaptertypes.WELConfig wg sync.WaitGroup isRunning uint32 mRunning sync.RWMutex @@ -29,7 +30,7 @@ type WELAdapter struct { hSubs []EVT_HANDLE } -func NewWELAdapter(conf WELConfig) (*WELAdapter, chan struct{}, error) { +func NewWELAdapter(conf adaptertypes.WELConfig) (*WELAdapter, chan struct{}, error) { a := &WELAdapter{ conf: conf, isRunning: 1, diff --git a/wel/conf.go b/wel/conf.go deleted file mode 100644 index 55e89d4..0000000 --- a/wel/conf.go +++ /dev/null @@ -1,11 +0,0 @@ -package usp_wel - -import ( - "github.com/refractionPOINT/go-uspclient" -) - -type WELConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - EvtSources string `json:"evt_sources,omitempty" yaml:"evt_sources,omitempty"` - WriteTimeoutSec uint64 `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty"` -} diff --git a/wel/noop.go b/wel/noop.go index fdab874..81bf94c 100644 --- a/wel/noop.go +++ b/wel/noop.go @@ -3,7 +3,11 @@ package usp_wel -import "errors" +import ( + "errors" + + "github.com/refractionPOINT/usp-adapters/adaptertypes" +) // Dummy noop file to build when the platform // is _not_ Windows since the WEL API is only @@ -11,7 +15,7 @@ import "errors" type WELAdapter struct{} -func NewWELAdapter(conf WELConfig) (*WELAdapter, chan struct{}, error) { +func NewWELAdapter(conf adaptertypes.WELConfig) (*WELAdapter, chan struct{}, error) { return nil, nil, errors.New("wel (Windows Event Logs) collection not supported outside of Windows") } diff --git a/wiz/client.go b/wiz/client.go index 6f2a041..bf9a6b4 100644 --- a/wiz/client.go +++ b/wiz/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -14,11 +13,12 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) type WizAdapter struct { - conf WizConfig + conf adaptertypes.WizConfig uspClient *uspclient.Client httpClient *http.Client @@ -31,47 +31,7 @@ type WizAdapter struct { expiresAt time.Time } -type WizConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ClientID string `json:"client_id" yaml:"client_id"` - ClientSecret string `json:"client_secret" yaml:"client_secret"` - URL string `json:"url" yaml:"url"` - Query string `json:"query" yaml:"query"` - Variables map[string]interface{} `json:"variables" yaml:"variables"` - TimeField string `json:"time_field" yaml:"time_field"` // e.g., "createdAt", "updatedAt" - DataPath []string `json:"data_path" yaml:"data_path"` // e.g., ["data", "securityIssues", "issues"] - IDField string `json:"id_field" yaml:"id_field"` // e.g., "id" -} - -func (c *WizConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ClientID == "" { - return errors.New("missing client_id") - } - if c.ClientSecret == "" { - return errors.New("missing client_secret") - } - if c.URL == "" { - return errors.New("missing url") - } - if c.Query == "" { - return errors.New("missing query") - } - if c.TimeField == "" { - return errors.New("missing time_field") - } - if len(c.DataPath) == 0 { - return errors.New("missing data_path") - } - if c.IDField == "" { - return errors.New("missing id_field") - } - return nil -} - -func NewWizAdapter(conf WizConfig) (*WizAdapter, chan struct{}, error) { +func NewWizAdapter(conf adaptertypes.WizConfig) (*WizAdapter, chan struct{}, error) { var err error a := &WizAdapter{ conf: conf, diff --git a/zendesk/client.go b/zendesk/client.go index 33757d1..1239683 100644 --- a/zendesk/client.go +++ b/zendesk/client.go @@ -4,7 +4,6 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -14,6 +13,7 @@ import ( "github.com/refractionPOINT/go-uspclient" "github.com/refractionPOINT/go-uspclient/protocol" + "github.com/refractionPOINT/usp-adapters/adaptertypes" "github.com/refractionPOINT/usp-adapters/utils" ) @@ -29,7 +29,7 @@ type opRequest struct { } type ZendeskAdapter struct { - conf ZendeskConfig + conf adaptertypes.ZendeskConfig uspClient *uspclient.Client httpClient *http.Client @@ -42,31 +42,7 @@ type ZendeskAdapter struct { dedupe map[string]int64 } -type ZendeskConfig struct { - ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"` - ApiToken string `json:"api_token" yaml:"api_token"` - ZendeskDomain string `json:"zendesk_domain" yaml:"zendesk_domain"` - ZendeskEmail string `json:"zendesk_email" yaml:"zendesk_email"` -} - -func (c *ZendeskConfig) Validate() error { - if err := c.ClientOptions.Validate(); err != nil { - return fmt.Errorf("client_options: %v", err) - } - if c.ApiToken == "" { - return errors.New("missing api token") - } - if c.ZendeskDomain == "" { - return errors.New("missing zendesk domain (e.g., 'your-company.zendesk.com')") - } - if c.ZendeskEmail == "" { - return errors.New("missing zendesk email") - } - - return nil -} - -func NewZendeskAdapter(conf ZendeskConfig) (*ZendeskAdapter, chan struct{}, error) { +func NewZendeskAdapter(conf adaptertypes.ZendeskConfig) (*ZendeskAdapter, chan struct{}, error) { var err error a := &ZendeskAdapter{ conf: conf,