From 57f76c18b1842fcee3e34a5b3611ec6a5c9b641a Mon Sep 17 00:00:00 2001
From: Arik Kfir <arik@kfirs.com>
Date: Sun, 14 Jul 2024 14:31:12 +0300
Subject: [PATCH] feat: support sending logs to GCP

This change adds support for sending logs to GCP Logging, for easier
inspection after failed CI runs, or even for local development logs.
---
 .github/workflows/build.yaml                        |  6 ++++++
 Makefile                                            |  8 +++++++-
 .../observability/otel/collector/agent-values.yaml  | 13 +++++++++++++
 .../local/security/gcp-service-account/.gitignore   |  1 +
 .../security/gcp-service-account/kustomization.yaml | 11 +++++++++++
 scripts/setup-observability.sh                      |  2 ++
 6 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 deploy/local/security/gcp-service-account/.gitignore
 create mode 100644 deploy/local/security/gcp-service-account/kustomization.yaml

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 2408bb7..9660e8b 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -102,6 +102,12 @@ jobs:
       - run: echo -n ${GITHUB_TOKEN} | helm registry login ghcr.io --username ${{ github.actor }} --password-stdin
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: |
+          cat > deploy/local/security/gcp-service-account/service-account-key.json <<EOF
+          ${GCP_SERVICE_ACCOUNT_KEY}
+          EOF
+        env:
+          GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_OTEL_SERVICE_ACCOUNT_KEY }}
       - run: make setup-observability
       - run: |
           helm install local-devbot oci://ghcr.io/arikkfir/devbot --version ${VERSION} \
diff --git a/Makefile b/Makefile
index 32af8f9..e90fb05 100644
--- a/Makefile
+++ b/Makefile
@@ -17,10 +17,16 @@ ifdef E2E_VERBOSITY # Set E2E_VERBOSITY to "-v" or "-vv" for extra output
 else
 	GINKGO_FLAGS := $(GINKGO_FLAGS) --succinct
 endif
+GCP_PROJECT := $(shell gcloud config get core/project)
 #-----------------------------------------------------------------------------------------------------------------------
 
+deploy/local/security/gcp-service-account/service-account-key.json:
+	gcloud iam service-accounts keys create \
+		./deploy/local/security/gcp-service-account/service-account-key.json \
+		--iam-account=otel-collector@$(GCP_PROJECT).iam.gserviceaccount.com
+
 .PHONY: setup
-setup:
+setup: deploy/local/security/gcp-service-account/service-account-key.json
 	npm install -g smee-client
 	go mod download -x
 	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0
diff --git a/deploy/local/observability/otel/collector/agent-values.yaml b/deploy/local/observability/otel/collector/agent-values.yaml
index 114fb74..b442e3d 100644
--- a/deploy/local/observability/otel/collector/agent-values.yaml
+++ b/deploy/local/observability/otel/collector/agent-values.yaml
@@ -17,6 +17,8 @@ ports:
     containerPort: 9090
     protocol: TCP
 presets:
+  logsCollection:
+    enabled: true
   kubernetesAttributes:
     enabled: true
     extractAllPodLabels: true
@@ -54,6 +56,11 @@ config:
                 replacement: $$1:$$2
                 target_label: __address__
   processors:
+    attributes/clusterid:
+      actions:
+        - key: cluster_id
+          value: local
+          action: insert
     k8sattributes:
       extract:
         metadata:
@@ -123,8 +130,14 @@ config:
   exporters:
     prometheus:
       endpoint: "0.0.0.0:9090"
+    googlecloud:
+      log:
+        default_log_name: "devbot.kfirs.com/dev"
   service:
     pipelines:
+      logs:
+        processors: [ attributes/clusterid ]
+        exporters: [ googlecloud ]
       metrics:
         receivers: [ prometheus ]
         processors: [ transform/prometheus-labels ]
diff --git a/deploy/local/security/gcp-service-account/.gitignore b/deploy/local/security/gcp-service-account/.gitignore
new file mode 100644
index 0000000..a6c57f5
--- /dev/null
+++ b/deploy/local/security/gcp-service-account/.gitignore
@@ -0,0 +1 @@
+*.json
diff --git a/deploy/local/security/gcp-service-account/kustomization.yaml b/deploy/local/security/gcp-service-account/kustomization.yaml
new file mode 100644
index 0000000..fb02463
--- /dev/null
+++ b/deploy/local/security/gcp-service-account/kustomization.yaml
@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+generatorOptions:
+  disableNameSuffixHash: true
+  immutable: true
+secretGenerator:
+  - name: otel-collector-gcp
+    type: Opaque
+    files:
+      - service-account-key.json
diff --git a/scripts/setup-observability.sh b/scripts/setup-observability.sh
index 847480e..d688b6a 100755
--- a/scripts/setup-observability.sh
+++ b/scripts/setup-observability.sh
@@ -32,10 +32,12 @@ helm upgrade --install prometheus prometheus --repo https://prometheus-community
   --values ./deploy/local/observability/prometheus/prometheus-values.yaml
 
 # Install the OTEL collector agent
+kubectl apply -k deploy/local/security/gcp-service-account
 helm upgrade --install otel-agent opentelemetry-collector --repo https://open-telemetry.github.io/opentelemetry-helm-charts \
   --namespace observability \
   --description "Agent (daemonset) OTEL collector for pull telemetry." \
   --values ./deploy/local/observability/otel/collector/agent-values.yaml \
+  --set "config.exporters.googlecloud.project=$(gcloud config get core/project)" \
   --version "${OTEL_CHART_VERSION}"
 helm upgrade --install otel-gateway opentelemetry-collector --repo https://open-telemetry.github.io/opentelemetry-helm-charts \
   --namespace observability \