Refactor Performance Tests and add Grafana, Pyroscope and Prometheus monitoring stack

.gitignore

@@ -2,6 +2,8 @@ vendor/
 apiserver.local.config/
 /apiserver/porch
 *.log
+*.csv
+load_test_results.txt
 .env
 
 # Development artifact path

cmd/porch/main.go

@@ -16,7 +16,9 @@ package main
 
 import (
 	"os"
+	"time"
 
+	"github.com/nephio-project/porch/internal/metrics"
 	porchotel "github.com/nephio-project/porch/internal/otel"
 	"github.com/nephio-project/porch/pkg/cmd/server"
 	genericapiserver "k8s.io/apiserver/pkg/server"
@@ -34,12 +36,26 @@ func main() {
 func run() int {
 	log.SetLogger(zap.New(zap.UseDevMode(true)))
 	ctx := genericapiserver.SetupSignalContext()
-	err := porchotel.SetupOpenTelemetry(ctx)
+	otelResources, err := porchotel.SetupOpenTelemetry(ctx)
 	if err != nil {
 		genericapiserver.RequestShutdown()
 		klog.Errorf("%v\n", err)
 		return 1
 	}
+	defer func() {
+		if err := otelResources.ShutdownWithTimeout(10 * time.Second); err != nil {
+			klog.Warningf("failed to gracefully shutdown OpenTelemetry: %v", err)
+		}
+	}()
+
+	prof := &metrics.Profiling{}
+	prof.Start()
+	defer prof.Stop()
+
+	pyro := &metrics.PyroscopeProfiling{}
+	pyro.Start()
+	defer pyro.Stop()
+
 	options := server.NewPorchServerOptions(os.Stdout, os.Stderr)
 	cmd := server.NewCommandStartPorchServer(ctx, options)
 	code := cli.Run(cmd)

controllers/main.go

@@ -25,7 +25,9 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
+	"github.com/nephio-project/porch/internal/metrics"
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
@@ -122,6 +124,7 @@ func run(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	metrics.InitMetrics()
 
 	mgr, err := newManager(ctx, scheme)
 	if err != nil {
@@ -203,9 +206,15 @@ func newManager(ctx context.Context, scheme *runtime.Scheme) (ctrl.Manager, erro
 	}
 
 	otel.SetLogger(klog.NewKlogr())
-	if err := porchotel.SetupOpenTelemetry(ctx); err != nil {
+	otelResources, err := porchotel.SetupOpenTelemetry(ctx)
+	if err != nil {
 		return nil, fmt.Errorf("error setting up OpenTelemetry: %w", err)
 	}
+	defer func() {
+		if shutdownErr := otelResources.ShutdownWithTimeout(10 * time.Second); shutdownErr != nil {
+			klog.Warningf("failed to gracefully shutdown OpenTelemetry: %v", shutdownErr)
+		}
+	}()
 
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme: scheme,
@@ -323,7 +332,6 @@ func setupFunctionConfigReconciler(mgr ctrl.Manager) (*reconciler.FunctionConfig
 	return functionConfigStore, nil
 }
 
-
 // --- Helpers ---
 
 func buildReconcilerMap(reconcilers ...Reconciler) map[string]Reconciler {

controllers/packagerevisions/pkg/controllers/packagerevision/render.go

@@ -150,7 +150,6 @@ func renderTrigger(pr *porchv1alpha2.PackageRevision) (requested string, annotat
 	return
 }
 
-
 // isRenderStale returns true if the annotation changed during render.
 func isRenderStale(currentAnnotation, rendered string) bool {
 	return currentAnnotation != rendered

controllers/repositories/pkg/controllers/repository/repository_controller.go

@@ -67,7 +67,7 @@ type RepositoryReconciler struct {
 
 	// Private implementation details
 	syncLimiter    chan struct{} // Semaphore for sync concurrency
-	coldStartRepos sync.Map     // Tracks repos that have synced since startup
+	coldStartRepos sync.Map      // Tracks repos that have synced since startup
 }
 
 //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 rbac:headerFile=../../../../../scripts/boilerplate.yaml.txt,roleName=porch-controllers-repositories,year=$YEAR_GEN webhook paths="." output:rbac:artifacts:config=../../../config/rbac