feat: support sending logs to GCP

.github/workflows/build.yaml

@@ -102,6 +102,12 @@ jobs:
       - run: echo -n ${GITHUB_TOKEN} | helm registry login ghcr.io --username ${{ github.actor }} --password-stdin
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: |
+          cat > deploy/local/security/gcp-service-account/service-account-key.json <<EOF
+          ${GCP_SERVICE_ACCOUNT_KEY}
+          EOF
+        env:
+          GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_OTEL_SERVICE_ACCOUNT_KEY }}
       - run: make setup-observability
       - run: |
           helm install local-devbot oci://ghcr.io/arikkfir/devbot --version ${VERSION} \

Makefile

@@ -17,10 +17,16 @@ ifdef E2E_VERBOSITY # Set E2E_VERBOSITY to "-v" or "-vv" for extra output
 else
 	GINKGO_FLAGS := $(GINKGO_FLAGS) --succinct
 endif
+GCP_PROJECT := $(shell gcloud config get core/project)
 #-----------------------------------------------------------------------------------------------------------------------
 
+deploy/local/security/gcp-service-account/service-account-key.json:
+	gcloud iam service-accounts keys create \
+		./deploy/local/security/gcp-service-account/service-account-key.json \
+		--iam-account=otel-collector@$(GCP_PROJECT).iam.gserviceaccount.com
+
 .PHONY: setup
-setup:
+setup: deploy/local/security/gcp-service-account/service-account-key.json
 	npm install -g smee-client
 	go mod download -x
 	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0

deploy/local/observability/otel/collector/agent-values.yaml

@@ -17,6 +17,8 @@ ports:
     containerPort: 9090
     protocol: TCP
 presets:
+  logsCollection:
+    enabled: true
   kubernetesAttributes:
     enabled: true
     extractAllPodLabels: true
@@ -54,6 +56,11 @@ config:
                 replacement: $$1:$$2
                 target_label: __address__
   processors:
+    attributes/clusterid:
+      actions:
+        - key: cluster_id
+          value: local
+          action: insert
     k8sattributes:
       extract:
         metadata:
@@ -123,8 +130,14 @@ config:
   exporters:
     prometheus:
       endpoint: "0.0.0.0:9090"
+    googlecloud:
+      log:
+        default_log_name: "devbot.kfirs.com/dev"
   service:
     pipelines:
+      logs:
+        processors: [ attributes/clusterid ]
+        exporters: [ googlecloud ]
       metrics:
         receivers: [ prometheus ]
         processors: [ transform/prometheus-labels ]

deploy/local/security/gcp-service-account/.gitignore

@@ -0,0 +1 @@
+*.json

deploy/local/security/gcp-service-account/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+generatorOptions:
+  disableNameSuffixHash: true
+  immutable: true
+secretGenerator:
+  - name: otel-collector-gcp
+    type: Opaque
+    files:
+      - service-account-key.json

scripts/setup-observability.sh

@@ -32,10 +32,12 @@ helm upgrade --install prometheus prometheus --repo https://prometheus-community
   --values ./deploy/local/observability/prometheus/prometheus-values.yaml
 
 # Install the OTEL collector agent
+kubectl apply -k deploy/local/security/gcp-service-account
 helm upgrade --install otel-agent opentelemetry-collector --repo https://open-telemetry.github.io/opentelemetry-helm-charts \
   --namespace observability \
   --description "Agent (daemonset) OTEL collector for pull telemetry." \
   --values ./deploy/local/observability/otel/collector/agent-values.yaml \
+  --set "config.exporters.googlecloud.project=$(gcloud config get core/project)" \
   --version "${OTEL_CHART_VERSION}"
 helm upgrade --install otel-gateway opentelemetry-collector --repo https://open-telemetry.github.io/opentelemetry-helm-charts \
   --namespace observability \