diff --git a/cmd/google_cloud_ops_agent_engine/main.go b/cmd/google_cloud_ops_agent_engine/main.go index 6fce0c5916..6be7134120 100644 --- a/cmd/google_cloud_ops_agent_engine/main.go +++ b/cmd/google_cloud_ops_agent_engine/main.go @@ -36,12 +36,12 @@ var ( healthChecks = flag.Bool("healthchecks", false, "run health checks and exit") ) -func runHealthChecks() { +func runHealthChecks(otlpExporterEnabled bool) { logger := healthchecks.CreateHealthChecksLogger(*logsDir) defaultLogger := logs.NewSimpleLogger() - healthCheckResults := healthchecks.HealthCheckRegistryFactory().RunAllHealthChecks(logger) + healthCheckResults := healthchecks.HealthCheckRegistryFactory(otlpExporterEnabled).RunAllHealthChecks(logger) healthchecks.LogHealthCheckResults(healthCheckResults, defaultLogger) } @@ -59,7 +59,6 @@ func run() error { if err != nil { return err } - // Log the built-in and merged config files to STDOUT. These are then written // by journald to var/log/syslog and so to Cloud Logging once the ops-agent is // running. @@ -68,7 +67,7 @@ func run() error { switch *service { case "": - runHealthChecks() + runHealthChecks(uc.Global.GetOtlpExporter()) log.Println("Startup checks finished") if *healthChecks { // If healthchecks is set, stop here diff --git a/cmd/ops_agent_uap_plugin/plugin.go b/cmd/ops_agent_uap_plugin/plugin.go index 8bfec45e37..314a54b70e 100644 --- a/cmd/ops_agent_uap_plugin/plugin.go +++ b/cmd/ops_agent_uap_plugin/plugin.go @@ -241,8 +241,8 @@ func writeCustomConfigToFile(req *pb.StartRequest, configPath string) error { return nil } -func runHealthChecks(healthCheckFileLogger logs.StructuredLogger) { - gceHealthChecks := healthchecks.HealthCheckRegistryFactory() +func runHealthChecks(healthCheckFileLogger logs.StructuredLogger, otlpExporterEnabled bool) { + gceHealthChecks := healthchecks.HealthCheckRegistryFactory(otlpExporterEnabled) // Log health check results to health-checks.log log file. gceHealthChecks.RunAllHealthChecks(healthCheckFileLogger) diff --git a/cmd/ops_agent_uap_plugin/service_linux.go b/cmd/ops_agent_uap_plugin/service_linux.go index f532ba5cd9..8c8fb50819 100644 --- a/cmd/ops_agent_uap_plugin/service_linux.go +++ b/cmd/ops_agent_uap_plugin/service_linux.go @@ -103,14 +103,15 @@ func (ps *OpsAgentPluginServer) Start(ctx context.Context, msg *pb.StartRequest) } // Ops Agent config validation - if err := validateOpsAgentConfig(pContext, OpsAgentConfigLocationLinux); err != nil { + uc, err := validateOpsAgentConfig(pContext, OpsAgentConfigLocationLinux) + if err != nil { ps.cancelAndSetPluginError(&OpsAgentPluginError{Message: fmt.Sprintf("Start() failed to validate the custom Ops Agent config: %s", err), ShouldRestart: false}) return &pb.StartResponse{}, nil } // Trigger Healthchecks. healthCheckFileLogger := healthchecks.CreateHealthChecksLogger(filepath.Join(pluginStateDir, LogsDirectory)) - runHealthChecks(healthCheckFileLogger) + runHealthChecks(healthCheckFileLogger, uc.Global.GetOtlpExporter()) // Subagent config generation if err := generateSubagentConfigs(pContext, ps.runCommand, pluginInstallDir, pluginStateDir); err != nil { @@ -198,9 +199,8 @@ func runCommand(cmd *exec.Cmd) (string, error) { return string(out), err } -func validateOpsAgentConfig(ctx context.Context, opsAgentConfigLocation string) error { - _, err := confgenerator.MergeConfFiles(ctx, opsAgentConfigLocation) - return err +func validateOpsAgentConfig(ctx context.Context, opsAgentConfigLocation string) (*confgenerator.UnifiedConfig, error) { + return confgenerator.MergeConfFiles(ctx, opsAgentConfigLocation) } func generateSubagentConfigs(ctx context.Context, runCommand RunCommandFunc, pluginInstallDirectory string, pluginStateDirectory string) error { diff --git a/cmd/ops_agent_uap_plugin/service_linux_test.go b/cmd/ops_agent_uap_plugin/service_linux_test.go index 95fb709df8..77171414bf 100644 --- a/cmd/ops_agent_uap_plugin/service_linux_test.go +++ b/cmd/ops_agent_uap_plugin/service_linux_test.go @@ -126,7 +126,7 @@ func Test_validateOpsAgentConfig(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() - err := validateOpsAgentConfig(ctx, tc.path) + _, err := validateOpsAgentConfig(ctx, tc.path) gotSuccess := (err == nil) if gotSuccess != tc.wantSuccess { t.Errorf("%s: validateOpsAgentConfig() got success = %v, want %v, error: %v", tc.name, gotSuccess, tc.wantSuccess, err) diff --git a/cmd/ops_agent_uap_plugin/service_windows.go b/cmd/ops_agent_uap_plugin/service_windows.go index c1b4d2e689..0f6615986f 100644 --- a/cmd/ops_agent_uap_plugin/service_windows.go +++ b/cmd/ops_agent_uap_plugin/service_windows.go @@ -111,7 +111,8 @@ func (ps *OpsAgentPluginServer) Start(ctx context.Context, msg *pb.StartRequest) } // Subagents config validation and generation. - if err := generateSubAgentConfigs(ctx, OpsAgentConfigLocationWindows, pluginStateDir); err != nil { + uc, err := generateSubAgentConfigs(ctx, OpsAgentConfigLocationWindows, pluginStateDir) + if err != nil { ps.cancelAndSetPluginError(&OpsAgentPluginError{ Message: fmt.Sprintf("Start() failed to validate the custom Ops Agent config, and generate sub-agents config: %s", err), ShouldRestart: false, @@ -121,7 +122,7 @@ func (ps *OpsAgentPluginServer) Start(ctx context.Context, msg *pb.StartRequest) // Trigger Healthchecks. healthCheckFileLogger := healthchecks.CreateHealthChecksLogger(filepath.Join(pluginStateDir, LogsDirectory)) - runHealthChecks(healthCheckFileLogger) + runHealthChecks(healthCheckFileLogger, uc.Global.GetOtlpExporter()) // Create a Windows Job object and stores its handle, to ensure that all child processes are killed when the parent process exits. _, err = createWindowsJobHandle() @@ -205,10 +206,10 @@ func findPreExistentAgents(mgr serviceManager, agentWindowsServiceNames []string return alreadyInstalledAgentServiceNames, nil } -func generateSubAgentConfigs(ctx context.Context, userConfigPath string, pluginStateDir string) error { +func generateSubAgentConfigs(ctx context.Context, userConfigPath string, pluginStateDir string) (*confgenerator.UnifiedConfig, error) { uc, err := confgenerator.MergeConfFiles(ctx, userConfigPath) if err != nil { - return err + return nil, err } log.Printf("Built-in config:\n%s\n", confgenerator.BuiltInConfStructs["windows"]) @@ -216,7 +217,7 @@ func generateSubAgentConfigs(ctx context.Context, userConfigPath string, pluginS // The generated otlp metric json files are used only by the otel service. if err = self_metrics.GenerateOpsAgentSelfMetricsOTLPJSON(ctx, userConfigPath, filepath.Join(pluginStateDir, GeneratedConfigsOutDir, "otel")); err != nil { - return err + return nil, err } for _, subagent := range []string{ @@ -229,10 +230,10 @@ func generateSubAgentConfigs(ctx context.Context, userConfigPath string, pluginS filepath.Join(pluginStateDir, LogsDirectory), filepath.Join(pluginStateDir, RuntimeDirectory), filepath.Join(pluginStateDir, GeneratedConfigsOutDir, subagent)); err != nil { - return err + return nil, err } } - return nil + return uc, nil } func createWindowsJobHandle() (windows.Handle, error) { diff --git a/cmd/ops_agent_uap_plugin/service_windows_test.go b/cmd/ops_agent_uap_plugin/service_windows_test.go index ef2cf26d3c..f742c1a903 100644 --- a/cmd/ops_agent_uap_plugin/service_windows_test.go +++ b/cmd/ops_agent_uap_plugin/service_windows_test.go @@ -155,7 +155,7 @@ func Test_runHealthChecks_LogFileNonEmpty(t *testing.T) { defer os.Remove(healthCheckLogFile.Name()) mockHealthCheckLogger := &mockHealthCheckLogger{logFile: healthCheckLogFile} - runHealthChecks(mockHealthCheckLogger) + runHealthChecks(mockHealthCheckLogger, false) // Check if the log file has content fileInfo, err := os.Stat(healthCheckLogFile.Name()) @@ -204,7 +204,7 @@ func Test_generateSubAgentConfigs(t *testing.T) { } userConfigFile.Close() - err = generateSubAgentConfigs(ctx, userConfigFile.Name(), tc.pluginStateDir) + _, err = generateSubAgentConfigs(ctx, userConfigFile.Name(), tc.pluginStateDir) if (err != nil) != tc.wantError { t.Errorf("generateSubAgentConfigs() returned error: %v, want error: %v", err, tc.wantError) } diff --git a/cmd/ops_agent_windows/main_windows.go b/cmd/ops_agent_windows/main_windows.go index 44bd4403c5..90ab04eb66 100644 --- a/cmd/ops_agent_windows/main_windows.go +++ b/cmd/ops_agent_windows/main_windows.go @@ -15,12 +15,14 @@ package main import ( + "context" "flag" "fmt" "log" "os" "path/filepath" + "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/internal/healthchecks" "github.com/GoogleCloudPlatform/ops-agent/internal/logs" "github.com/kardianos/osext" @@ -63,7 +65,20 @@ func main() { } infoLog.Printf("uninstalled services") } else if *healthChecks { - healthCheckResults := getHealthCheckResults() + ctx := context.Background() + base, err := osext.ExecutableFolder() + if err != nil { + log.Fatalf("failed to determine executable folder: %v", err) + } + configPath := filepath.Join(base, "../config/config.yaml") + otlpExporterEnabled := false + uc, err := confgenerator.MergeConfFiles(ctx, configPath) + if err == nil { + otlpExporterEnabled = uc.Global.GetOtlpExporter() + } else { + log.Printf("failed to load config: %v", err) + } + healthCheckResults := getHealthCheckResults(otlpExporterEnabled) healthchecks.LogHealthCheckResults(healthCheckResults, infoLog) infoLog.Println("Health checks finished") } else { diff --git a/cmd/ops_agent_windows/run_windows.go b/cmd/ops_agent_windows/run_windows.go index 17b3df7206..cc1e3e971f 100644 --- a/cmd/ops_agent_windows/run_windows.go +++ b/cmd/ops_agent_windows/run_windows.go @@ -53,6 +53,7 @@ type service struct { log debug.Log userConf string outDirectory string + uc *confgenerator.UnifiedConfig } func (s *service) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) { @@ -141,16 +142,16 @@ func (s *service) checkForStandaloneAgents(unified *confgenerator.UnifiedConfig) return nil } -func getHealthCheckResults() []healthchecks.HealthCheckResult { +func getHealthCheckResults(otlpExporterEnabled bool) []healthchecks.HealthCheckResult { logsDir := filepath.Join(os.Getenv("PROGRAMDATA"), dataDirectory, "log") - gceHealthChecks := healthchecks.HealthCheckRegistryFactory() + gceHealthChecks := healthchecks.HealthCheckRegistryFactory(otlpExporterEnabled) logger := healthchecks.CreateHealthChecksLogger(logsDir) return gceHealthChecks.RunAllHealthChecks(logger) } func (srv *service) runHealthChecks() { - healthCheckResults := getHealthCheckResults() + healthCheckResults := getHealthCheckResults(srv.uc.Global.GetOtlpExporter()) logger := logs.WindowsServiceLogger{EventID: EngineEventID, Logger: srv.log} healthchecks.LogHealthCheckResults(healthCheckResults, logger) srv.log.Info(EngineEventID, "Startup checks finished") @@ -162,6 +163,7 @@ func (s *service) generateConfigs(ctx context.Context) error { if err != nil { return err } + s.uc = uc s.log.Info(EngineEventID, fmt.Sprintf("Built-in config:\n%s\n", confgenerator.BuiltInConfStructs["windows"])) s.log.Info(EngineEventID, fmt.Sprintf("Merged config:\n%s\n", uc)) diff --git a/confgenerator/agentmetrics.go b/confgenerator/agentmetrics.go index 5ddb372f39..c533d41aad 100644 --- a/confgenerator/agentmetrics.go +++ b/confgenerator/agentmetrics.go @@ -22,7 +22,6 @@ import ( "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel/ottl" - "github.com/GoogleCloudPlatform/ops-agent/internal/experiments" ) // AgentSelfMetrics provides the agent.googleapis.com/agent/ metrics. @@ -34,6 +33,7 @@ type AgentSelfMetrics struct { FluentBitPort int OtelPort int OtelRuntimeDir string + OtlpExporterEnabled bool } // Following reference : https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto @@ -182,7 +182,7 @@ func (r AgentSelfMetrics) OtelPipelineProcessors(ctx context.Context) []otel.Com `metric.name == "` + durationCountMetric + `" and (not IsMatch(datapoint.attributes["grpc.target"], "monitoring.googleapis"))`, }) - expOtlpExporter := experiments.FromContext(ctx)["otlp_exporter"] + expOtlpExporter := r.OtlpExporterEnabled var extraTransforms []map[string]interface{} if expOtlpExporter { durationMetric = "rpc.client.call.duration" @@ -293,7 +293,7 @@ func (r AgentSelfMetrics) LoggingMetricsPipelineProcessors(ctx context.Context) otel.AggregateLabels("sum", "response_code"), ) - expOtlpExporter := experiments.FromContext(ctx)["otlp_exporter"] + expOtlpExporter := r.OtlpExporterEnabled if expOtlpExporter { durationMetric = "rpc.client.call.duration" durationCountMetric = "rpc.client.call.duration_count" diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go index 9c9e0e1f78..445087f4d0 100644 --- a/confgenerator/confgenerator.go +++ b/confgenerator/confgenerator.go @@ -31,7 +31,6 @@ import ( "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/resourcedetector" - "github.com/GoogleCloudPlatform/ops-agent/internal/experiments" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" ) @@ -87,39 +86,51 @@ func googleCloudLoggingExporter() otel.Component { } func ConvertPrometheusExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline { - return ConvertToOtlpExporter(pipeline, ctx, true, false) + return pipeline } func ConvertGCMOtelExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline { - return ConvertToOtlpExporter(pipeline, ctx, false, false) + return pipeline } func ConvertGCMSystemExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline { - return ConvertToOtlpExporter(pipeline, ctx, false, true) + return pipeline } func ConvertToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context, isPrometheus bool, isSystem bool) otel.ReceiverPipeline { - expOtlpExporter := experiments.FromContext(ctx)["otlp_exporter"] - if !expOtlpExporter { - return pipeline - } - - if _, ok := pipeline.ExporterTypes["metrics"]; ok { - if isPrometheus { - pipeline.ExporterTypes["metrics"] = otel.GMP - } else { - pipeline.ExporterTypes["metrics"] = otel.OTLP_Metrics - if isSystem { - pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricsRemoveInstrumentationLibraryLabelsAttributes()) - pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricsRemoveServiceAttributes()) + return pipeline +} + +func ConvertPipelinesToOtlp(receiverPipelines map[string]otel.ReceiverPipeline) { + for rID, pipeline := range receiverPipelines { + if _, ok := pipeline.ExporterTypes["metrics"]; ok { + if pipeline.ExporterTypes["metrics"] == otel.System { + pipeline.ExporterTypes["metrics"] = otel.OTLP_Metrics + if pipeline.Processors == nil { + pipeline.Processors = make(map[string][]otel.Component) + } + pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], + otel.MetricsRemoveInstrumentationLibraryLabelsAttributes(), + otel.MetricsRemoveServiceAttributes(), + ) + } else if pipeline.ExporterTypes["metrics"] == otel.OTel { + pipeline.ExporterTypes["metrics"] = otel.OTLP_Metrics + } + } + + if _, ok := pipeline.ExporterTypes["logs"]; ok { + if pipeline.ExporterTypes["logs"] == otel.Logging { + pipeline.ExporterTypes["logs"] = otel.OTLP_Logs } } - } - if _, ok := pipeline.ExporterTypes["logs"]; ok { - pipeline.ExporterTypes["logs"] = otel.OTLP_Logs + if _, ok := pipeline.ExporterTypes["traces"]; ok { + if pipeline.ExporterTypes["traces"] == otel.OTel { + pipeline.ExporterTypes["traces"] = otel.OTLP_Traces + } + } + receiverPipelines[rID] = pipeline } - return pipeline } func otlpExporterForMetrics(userAgent string) otel.Component { @@ -168,6 +179,21 @@ func otlpExporterForLogs(userAgent string) otel.Component { } } +func otlpExporterForTraces(userAgent string) otel.Component { + return otel.Component{ + Type: "otlp_grpc", + Config: map[string]interface{}{ + "endpoint": "telemetry.googleapis.com:443", + // b/485538253: Use pick_first balancer until we can understand why round_robin is failing. + "balancer_name": "pick_first", + "auth": map[string]interface{}{ + "authenticator": "googleclientauth", + }, + "user_agent": userAgent, + }, + } +} + func googleManagedPrometheusExporter(userAgent string) otel.Component { return otel.Component{ Type: "googlemanagedprometheus", @@ -208,6 +234,7 @@ func fileStorageExtension(stateDir string) otel.Component { func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDir string) (string, error) { p := platform.FromContext(ctx) + userAgent, _ := p.UserAgent("Google-Cloud-Ops-Agent-Metrics") metricVersionLabel, _ := p.VersionLabel("google-cloud-ops-agent-metrics") loggingVersionLabel, _ := p.VersionLabel("google-cloud-ops-agent-logging") @@ -223,8 +250,13 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi FluentBitPort: int(uc.GetFluentBitMetricsPort()), OtelPort: int(uc.GetOtelMetricsPort()), OtelRuntimeDir: outDir, + OtlpExporterEnabled: uc.Global.GetOtlpExporter(), } agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines, ctx) + + if uc.Global.GetOtlpExporter() { + ConvertPipelinesToOtlp(receiverPipelines) + } resource, err := p.GetResource() if err != nil { return "", err @@ -268,6 +300,17 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi }, }, }, + otel.OTLP_Traces: { + Exporter: otlpExporterForTraces(userAgent), + UsedExtensions: []string{googleClientAuthExtensionType}, + ProcessorsByType: map[string][]otel.Component{ + "traces": { + // Note: resourcedetection processor is added automatically at the receiver pipeline level in otel/modular.go. + otel.GCPProjectID(resource.ProjectName()), + otel.BatchProcessor(200, 200, "200ms"), + }, + }, + }, otel.Logging: { Exporter: googleCloudLoggingExporter(), UsedExtensions: []string{fileStorageExtensionType}, @@ -486,6 +529,7 @@ func (uc *UnifiedConfig) generateOtelPipelines(ctx context.Context) (map[string] // It returns a map of filenames to file contents. func (uc *UnifiedConfig) GenerateFluentBitConfigs(ctx context.Context, logsDir string, stateDir string) (map[string]string, error) { userAgent, _ := platform.FromContext(ctx).UserAgent("Google-Cloud-Ops-Agent-Logging") + components, err := uc.generateFluentbitComponents(ctx, userAgent) if err != nil { return nil, err diff --git a/confgenerator/confgenerator_test.go b/confgenerator/confgenerator_test.go index 50cfeab3a3..0c1508e2a5 100644 --- a/confgenerator/confgenerator_test.go +++ b/confgenerator/confgenerator_test.go @@ -343,8 +343,7 @@ func generateConfigs(pc platformConfig, testDir string) (got map[string]string, func generateOtelConfigWithOtlpExporterEnabled(got map[string]string, pc platformConfig, testDir string, otelGeneratedConfig string) { experimentsOtlp := map[string]bool{ - "otlp_exporter": true, - "otel_logging": true, + "otel_logging": true, } ctxOtlp := experiments.ContextWithExperiments(pc.platform.TestContext(context.Background()), experimentsOtlp) @@ -353,6 +352,12 @@ func generateOtelConfigWithOtlpExporterEnabled(got map[string]string, pc platfor filepath.Join("testdata", testDir, inputFileName), ) if err == nil { + if mergedUcOtlp.Global == nil { + mergedUcOtlp.Global = &confgenerator.Global{} + } + enabled := true + mergedUcOtlp.Global.OtlpExporter = &enabled + otelGeneratedConfigOtlp, err := mergedUcOtlp.GenerateOtelConfig(ctxOtlp, "", "") if err == nil { got["otel_otlp_exporter.yaml"] = otelGeneratedConfigOtlp diff --git a/confgenerator/config.go b/confgenerator/config.go index c9692fbcb6..30d4c1aad3 100644 --- a/confgenerator/config.go +++ b/confgenerator/config.go @@ -860,6 +860,7 @@ type TracesService struct { func (uc *UnifiedConfig) Validate(ctx context.Context) error { if uc.Logging != nil { + if err := uc.ValidateLogging(); err != nil { return err } diff --git a/confgenerator/config_global.go b/confgenerator/config_global.go index 3aaeb8ac9d..ca24fafe4e 100644 --- a/confgenerator/config_global.go +++ b/confgenerator/config_global.go @@ -17,6 +17,14 @@ package confgenerator type Global struct { DefaultSelfLogFileCollection *bool `yaml:"default_self_log_file_collection,omitempty"` DefaultLogFileRotation *LogFileRotation `yaml:"default_self_log_file_rotation,omitempty"` + OtlpExporter *bool `yaml:"otlp_exporter,omitempty"` +} + +func (g *Global) GetOtlpExporter() bool { + if g != nil && g.OtlpExporter != nil { + return *g.OtlpExporter + } + return false } // Get whether self log collection should be enabled. Defaults to true if unset. diff --git a/confgenerator/feature_tracking.go b/confgenerator/feature_tracking.go index 2480b553f4..be8eb1b927 100644 --- a/confgenerator/feature_tracking.go +++ b/confgenerator/feature_tracking.go @@ -22,8 +22,6 @@ import ( "sort" "strconv" "strings" - - "github.com/GoogleCloudPlatform/ops-agent/internal/experiments" ) var ( @@ -76,9 +74,10 @@ type CustomFeatures interface { // tag will be used instead of value from UnifiedConfig. func ExtractFeatures(ctx context.Context, userUc, mergedUc *UnifiedConfig) ([]Feature, error) { allFeatures := getOverriddenDefaultPipelines(userUc) + allFeatures = append(allFeatures, getSelfLogCollection(userUc)) allFeatures = append(allFeatures, getOTelLoggingSupportedConfig(ctx, mergedUc)) - allFeatures = append(allFeatures, getOtlpExporterExperimentConfig(ctx)...) + allFeatures = append(allFeatures, getOtlpExporterFeatureConfig(mergedUc)...) var err error var tempTrackedFeatures []Feature @@ -506,9 +505,9 @@ func getSelfLogCollection(uc *UnifiedConfig) Feature { return feature } -func getOtlpExporterExperimentConfig(ctx context.Context) []Feature { +func getOtlpExporterFeatureConfig(uc *UnifiedConfig) []Feature { featureEnabled := "false" - if experiments.FromContext(ctx)["otlp_exporter"] { + if uc.Global.GetOtlpExporter() { featureEnabled = "true" } diff --git a/confgenerator/otel/modular.go b/confgenerator/otel/modular.go index 1035ce9074..c97af0512d 100644 --- a/confgenerator/otel/modular.go +++ b/confgenerator/otel/modular.go @@ -42,6 +42,7 @@ const ( GMP OTLP_Metrics OTLP_Logs + OTLP_Traces Logging ) const ( @@ -62,6 +63,8 @@ func (t ExporterType) Name() string { return "otlp_metrics" } else if t == OTLP_Logs { return "otlp_logs" + } else if t == OTLP_Traces { + return "otlp_traces" } else { panic("unknown ExporterType") } diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/EXPERIMENTAL_FEATURES b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/EXPERIMENTAL_FEATURES index 1254b2789d..96e7b5fd45 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/EXPERIMENTAL_FEATURES +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/EXPERIMENTAL_FEATURES @@ -1 +1 @@ -otlp_exporter,otlp_logging,otel_logging +otlp_logging,otel_logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml index b503e21f03..530f2ed584 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -863,6 +865,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1385,9 +1392,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml index 474b8906ef..a34c1ab7c9 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -834,6 +836,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1325,9 +1332,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml index 3774472b5e..ea53de73fc 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -896,6 +898,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1875,9 +1882,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml index 3774472b5e..ea53de73fc 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -896,6 +898,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1875,9 +1882,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/input.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/input.yaml index e8aec4319d..c67ff23062 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/input.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/input.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +global: + otlp_exporter: true combined: receivers: otlp: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/EXPERIMENTAL_FEATURES b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/EXPERIMENTAL_FEATURES index 91babc70cd..515983c71a 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/EXPERIMENTAL_FEATURES +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/EXPERIMENTAL_FEATURES @@ -1 +1 @@ -otlp_exporter,otlp_logging +otlp_logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml index 0e5054bdbb..0c19f1cd78 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -863,6 +865,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1270,9 +1277,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml index cc3e2470e0..7d921df420 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -834,6 +836,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1210,9 +1217,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml index e56d6565f2..66825277d8 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -896,6 +898,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1760,9 +1767,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml index e56d6565f2..66825277d8 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml @@ -1,12 +1,4 @@ exporters: - googlecloud/otel: - metric: - instrumentation_library_labels: true - prefix: "" - resource_filters: [] - service_resource_labels: true - skip_create_descriptor: true - user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) googlemanagedprometheus: metric: add_metric_suffixes: false @@ -35,6 +27,12 @@ exporters: balancer_name: pick_first endpoint: telemetry.googleapis.com:443 user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + otlp_grpc/otlp_traces: + auth: + authenticator: googleclientauth + balancer_name: pick_first + endpoint: telemetry.googleapis.com:443 + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) extensions: file_storage: create_directory: true @@ -48,6 +46,10 @@ processors: send_batch_max_size: 200 send_batch_size: 200 timeout: 200ms + batch/otlp_grpc/otlp_traces_traces_1: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 200ms cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -896,6 +898,11 @@ processors: - action: insert key: gcp.project_id value: test-project + resource/otlp_grpc/otlp_traces_traces_0: + attributes: + - action: insert + key: gcp.project_id + value: test-project resourcedetection/_global_0: detectors: - gcp @@ -1760,9 +1767,11 @@ service: - otlp/otlp traces/traces_otlp_otlp: exporters: - - googlecloud/otel + - otlp_grpc/otlp_traces processors: - resourcedetection/_global_1 + - resource/otlp_grpc/otlp_traces_traces_0 + - batch/otlp_grpc/otlp_traces_traces_1 receivers: - otlp/otlp telemetry: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/input.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/input.yaml index 7567f69f2b..51fea1c285 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/input.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/input.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +global: + otlp_exporter: true combined: receivers: otlp: diff --git a/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/EXPERIMENTAL_FEATURES b/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/EXPERIMENTAL_FEATURES deleted file mode 100644 index 9d450f37a8..0000000000 --- a/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/EXPERIMENTAL_FEATURES +++ /dev/null @@ -1 +0,0 @@ -otlp_exporter diff --git a/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/input.yaml b/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/input.yaml index 5474eeab87..bdcdd4f4a0 100644 --- a/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/input.yaml +++ b/confgenerator/testdata/goldens/invalid-metrics-exporter_otlphttp_invalid_processor/input.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +global: + otlp_exporter: true metrics: processors: metrics_filter: diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/EXPERIMENTAL_FEATURES b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/EXPERIMENTAL_FEATURES deleted file mode 100644 index 9d450f37a8..0000000000 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/EXPERIMENTAL_FEATURES +++ /dev/null @@ -1 +0,0 @@ -otlp_exporter diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/input.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/input.yaml index b9a8c4f33a..6dd6242de0 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/input.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/input.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +global: + otlp_exporter: true metrics: receivers: prometheus: diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua new file mode 100644 index 0000000000..dda27aee97 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua @@ -0,0 +1,28 @@ + +function process(tag, timestamp, record) +local v = "ops-agent"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentKind"] = value +end)(v) +local v = "latest"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentVersion"] = value +end)(v) +local v = "v1"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/schemaVersion"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua new file mode 100644 index 0000000000..c225be24d5 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua @@ -0,0 +1,19 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["severity"] +end)(); +(function(value) +record["severity"] = value +end)(nil); +local v = __field_0; +if v == "debug" then v = "DEBUG" +elseif v == "error" then v = "ERROR" +elseif v == "info" then v = "INFO" +elseif v == "warn" then v = "WARNING" +end +(function(value) +record["logging.googleapis.com/severity"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua new file mode 100644 index 0000000000..c5465182b5 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua @@ -0,0 +1,49 @@ + + function shallow_merge(record, parsedRecord) + -- If no exiting record exists + if (record == nil) then + return parsedRecord + end + + for k, v in pairs(parsedRecord) do + record[k] = v + end + + return record +end + +function merge(record, parsedRecord) + -- If no exiting record exists + if record == nil then + return parsedRecord + end + + -- Potentially overwrite or merge the original records. + for k, v in pairs(parsedRecord) do + -- If there is no conflict + if k == "logging.googleapis.com/logName" then + -- Ignore the parsed payload since the logName is controlled + -- by the OpsAgent. + elseif k == "logging.googleapis.com/labels" then + -- LogEntry.labels are basically a map[string]string and so only require a + -- shallow merge (one level deep merge). + record[k] = shallow_merge(record[k], v) + else + record[k] = v + end + end + + return record +end + +function parser_merge_record(tag, timestamp, record) + originalPayload = record["logging.googleapis.com/__tmp"] + if originalPayload == nil then + return 0, timestamp, record + end + + -- Remove original payload + record["logging.googleapis.com/__tmp"] = nil + record = merge(originalPayload, record) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua new file mode 100644 index 0000000000..6263563b66 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua @@ -0,0 +1,17 @@ + +function parser_nest(tag, timestamp, record) + local nestedRecord = {} + local parseKey = "message" + for k, v in pairs(record) do + if k ~= parseKey then + nestedRecord[k] = v + end + end + + local result = {} + result[parseKey] = record[parseKey] + result["logging.googleapis.com/__tmp"] = nestedRecord + + return 2, timestamp, result +end + diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/enabled_receivers_otlp.json b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/enabled_receivers_otlp.json new file mode 100644 index 0000000000..9cf0a64e2d --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/enabled_receivers_otlp.json @@ -0,0 +1 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/ops_agent/enabled_receivers","gauge":{"dataPoints":[{"attributes":[{"key":"telemetry_type","value":{"stringValue":"metrics"}},{"key":"receiver_type","value":{"stringValue":"hostmetrics"}}],"asInt":"1"},{"attributes":[{"key":"telemetry_type","value":{"stringValue":"logs"}},{"key":"receiver_type","value":{"stringValue":"files"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/feature_tracking_otlp.json new file mode 100644 index 0000000000..919ccdd5c6 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/feature_tracking_otlp.json @@ -0,0 +1 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"exporters:otlp"}},{"key":"key","value":{"stringValue":"otlp_exporter"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"exporters:otlp"}},{"key":"key","value":{"stringValue":"otlp_exporter"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/features.yaml new file mode 100644 index 0000000000..43639eb425 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/features.yaml @@ -0,0 +1,24 @@ +- module: logging + feature: service:pipelines + key: default_pipeline_overridden + value: "false" +- module: metrics + feature: service:pipelines + key: default_pipeline_overridden + value: "false" +- module: global + feature: default:self_log + key: default_self_log_file_collection + value: "true" +- module: logging + feature: service:otel_logging + key: otel_logging_supported_config + value: "true" +- module: metrics + feature: exporters:otlp + key: otlp_exporter + value: "true" +- module: logging + feature: exporters:otlp + key: otlp_exporter + value: "true" diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf new file mode 100644 index 0000000000..9d767d886b --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf @@ -0,0 +1,153 @@ +@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers +@SET logs_dir=/var/log/google-cloud-ops-agent + +[SERVICE] + Daemon off + Flush 1 + Log_Level info + dns.resolver legacy + storage.backlog.mem_limit 50M + storage.checksum off + storage.max_chunks_up 128 + storage.metrics on + storage.sync normal + +[INPUT] + Name fluentbit_metrics + Scrape_Interval 60 + Scrape_On_Start True + +[INPUT] + Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} + Interval_NSec 0 + Interval_Sec 600 + Name dummy + Tag ops-agent-health + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/ops-agent-fluent-bit + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path ${logs_dir}/subagents/logging-module.log + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag ops-agent-fluent-bit + storage.type memory + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/ops-agent-health + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path ${logs_dir}/health-checks.log + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag ops-agent-health + storage.type memory + +[FILTER] + Match ops-agent-fluent-bit + Name lua + call parser_nest + script b4a0dead382dce7b4fe011d3f59fdb6d.lua + +[FILTER] + Key_Name message + Match ops-agent-fluent-bit + Name parser + Preserve_Key True + Reserve_Data True + Parser ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing + +[FILTER] + Match ops-agent-fluent-bit + Name lua + call parser_merge_record + script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua + +[FILTER] + Match ops-agent-health + Name lua + call parser_nest + script b4a0dead382dce7b4fe011d3f59fdb6d.lua + +[FILTER] + Key_Name message + Match ops-agent-health + Name parser + Reserve_Data True + Parser ops-agent-health.health-checks-json + +[FILTER] + Match ops-agent-health + Name lua + call parser_merge_record + script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua + +[FILTER] + Match ops-agent-health + Name grep + Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug + +[FILTER] + Match ops-agent-fluent-bit + Name rewrite_tag + Rule message \[error\]\s\[lib\]\sbackend\sfailed ops-agent-health true + +[FILTER] + Name modify + Match ops-agent-health + Condition Key_value_matches message \[error\]\s\[lib\]\sbackend\sfailed + Set code LogPipelineErr + Set message "[Runtime Check] Result: FAIL, Error code: LogPipelineErr, Failure: Ops Agent logging pipeline failed, Solution: Refer to provided documentation link., Resource: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info" + +[FILTER] + Match ops-agent-fluent-bit + Name rewrite_tag + Rule message \[error\]\s\[parser\]\scannot\sparse ops-agent-health true + +[FILTER] + Name modify + Match ops-agent-health + Condition Key_value_matches message \[error\]\s\[parser\]\scannot\sparse + Set code LogParseErr + Set message "[Runtime Check] Result: WARNING, Error code: LogParseErr, Failure: Ops Agent failed to parse logs, Solution: Refer to provided documentation link., Resource: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info" + +[FILTER] + Match ops-agent-health + Name lua + call process + script 0f15dbe303dc7122d43443c9a4c31632.lua + +[FILTER] + Match ops-agent-* + Name lua + call process + script 4d6012ff003886818fb9b9285b4af962.lua + +[OUTPUT] + Match_Regex ^(ops-agent-health|ops-agent-fluent-bit)$ + Name stackdriver + Retry_Limit 3 + http_request_key logging.googleapis.com/httpRequest + net.connect_timeout_log_error False + resource gce_instance + stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + tls On + tls.verify Off + workers 8 + +[OUTPUT] + Match * + Name prometheus_exporter + host 0.0.0.0 + port 20202 diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf new file mode 100644 index 0000000000..3c603780bd --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf @@ -0,0 +1,13 @@ +[PARSER] + Format regex + Name ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing + Regex (?\[[ ]*(?