feat: initial implementation of bigquery IAM provisioning

.github/workflows/elixir-migration-check.yml

@@ -15,9 +15,9 @@ jobs:
     name: Get versions
     runs-on: ubuntu-latest
     outputs:
-      elixir:  ${{ steps.v.outputs.elixir }}
-      erlang:  ${{ steps.v.outputs.erlang }}
-      rust:    ${{ steps.v.outputs.rust }}
+      elixir: ${{ steps.v.outputs.elixir }}
+      erlang: ${{ steps.v.outputs.erlang }}
+      rust: ${{ steps.v.outputs.rust }}
     steps:
       - uses: actions/checkout@v6
       - name: Read versions from .tool-versions
@@ -33,7 +33,7 @@ jobs:
     needs: versions
     services:
       postgres:
-        image: bitnamilegacy/postgresql:13.15.0
+        image: bitnamilegacy/postgresql:15
         ports:
           - 5432:5432
         env:
@@ -74,6 +74,15 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
+      - name: Prepare Telegraf Output Dir
+        run: |
+          mkdir -p db/telegraf_output
+          touch db/telegraf_output/metrics.out
+          chmod 666 db/telegraf_output/metrics.out
+
+      - name: Start Telegraf
+        run: docker compose up -d telegraf
+
       - name: Set up Elixir
         uses: erlef/setup-beam@v1
         with:

Makefile

@@ -186,7 +186,10 @@ migrate:
 	@env $$(cat .dev.env | xargs) mix ecto.migrate
 
 
-.PHONY: __start__ migrate start.sb.pg start.sb.bq start.st.pg start.st.bq start.orange start.pink
+stripe:
+	stripe listen --forward-to localhost:4000/webhooks/stripe
+
+.PHONY: __start__ migrate stripe start.sb.pg start.sb.bq start.st.pg start.st.bq start.orange start.pink
 
 # Encryption and decryption of secrets
 # Usage:

config/dev.exs

@@ -80,5 +80,4 @@ config :logflare, Logflare.Cluster.Utils, min_cluster_size: 1
 config :open_api_spex, :cache_adapter, OpenApiSpex.Plug.NoneCache
 
 config :stripity_stripe,
-  api_key: "sk_test_thisisaboguskey",
-  api_base_url: "http://localhost:12111/v1/"
+  api_key: "sk_test_thisisaboguskey"

docker-compose.yml

@@ -10,10 +10,6 @@ services:
     volumes:
       - ./priv/setup.sql:/docker-entrypoint-initdb.d/setup.sql
       - pg-data:/var/lib/postgresql/data
-  stripe-mock:
-    image: stripe/stripe-mock:latest
-    ports:
-      - "12111:12111"
   base:
     build:
       dockerfile: ./Dockerfile.base
@@ -68,12 +64,12 @@ services:
       # - filebeat:/usr/share/filebeat/data
       # - /var/log/audit/:/var/log/audit/:ro
     # environment:
-      # - ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST:-node1}
-      # - KIBANA_HOST=${KIBANA_HOST:-node1}
-      # - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-elastic}
-      # - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-changeme}
-      # disable strict permission checks
-    command: [ "--strict.perms=false" ]
+    # - ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST:-node1}
+    # - KIBANA_HOST=${KIBANA_HOST:-node1}
+    # - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-elastic}
+    # - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-changeme}
+    # disable strict permission checks
+    command: ["--strict.perms=false"]
   loki:
     image: grafana/loki:3.1.0
     ports:
@@ -139,7 +135,7 @@ services:
       resources:
         limits:
           memory: 1500M
-          cpus: '1.0'
+          cpus: "1.0"
         reservations:
           memory: 512M
     shm_size: 256M

docs/docs.logflare.com/docs/concepts/endpoints.md

@@ -233,3 +233,12 @@ curl "https://api.logflare.app/api/endpoints/query/my-endpoint" \
   -H 'Content-Type: application/json; charset=utf-8'
 ```
 
+### Additional BigQuery Projects
+
+If the reservation is in a different GCP project than your primary BigQuery project, you must configure it as an additional BigQuery project under **Account > BigQuery Backend > Additional BigQuery Projects**.
+
+The main Logflare service account must be granted the following IAM roles on the additional project:
+
+- **Project IAM Admin** (`roles/resourcemanager.projectIamAdmin`) — required to set IAM policies for managed service accounts on the additional project
+- **BigQuery Admin** (`roles/bigquery.admin`) — required for managed service accounts to execute queries against reservations in the additional project
+

lib/logflare/backends/adaptor/bigquery_adaptor.ex

@@ -473,15 +473,18 @@ defmodule Logflare.Backends.Adaptor.BigQueryAdaptor do
   def update_iam_policy(user \\ nil) do
     CloudResourceManager.set_iam_policy(async: false)
 
-    if Map.get(user || %{}, :bigquery_project_id) do
-      # byob project, maybe append managed SA to policy
-      append_managed_sa_to_iam_policy(user)
+    if Map.get(user || %{}, :bigquery_project_id) ||
+         Map.get(user || %{}, :bigquery_additional_projects) do
+      # byob project or additional IAM projects, append managed SA to all policies
+      append_managed_sa_to_all_iam_projects(user)
     end
   end
 
   defdelegate get_iam_policy(user), to: CloudResourceManager
 
   defdelegate append_managed_sa_to_iam_policy(user), to: CloudResourceManager
+
+  defdelegate append_managed_sa_to_all_iam_projects(user), to: CloudResourceManager
   defdelegate append_managed_service_accounts(project_id, policy), to: CloudResourceManager
   defdelegate patch_dataset_access(user), to: Google.BigQuery
   defdelegate get_conn(conn_type), to: GenUtils

lib/logflare/google/cloud_resource_manager.ex

@@ -67,6 +67,54 @@ defmodule Logflare.Google.CloudResourceManager do
     end
   end
 
+  @spec append_managed_sa_to_all_iam_projects(User.t()) :: %{
+          primary: {:ok, Model.Policy.t()} | {:error, term()},
+          additional: [
+            {String.t(), {:ok, Model.Policy.t()} | {:error, term()}}
+          ]
+        }
+  def append_managed_sa_to_all_iam_projects(user) do
+    primary_result = append_managed_sa_to_iam_policy(user)
+
+    additional_results =
+      user
+      |> User.parse_bigquery_additional_projects()
+      |> Enum.map(fn project_id ->
+        {project_id, append_managed_sa_to_additional_project(user, project_id)}
+      end)
+
+    %{primary: primary_result, additional: additional_results}
+  end
+
+  defp append_managed_sa_to_additional_project(
+         %User{bigquery_enable_managed_service_accounts: false},
+         _project_id
+       ),
+       do: {:error, :managed_service_accounts_disabled}
+
+  defp append_managed_sa_to_additional_project(_user, project_id) do
+    with {:enabled?, true} <- {:enabled?, BigQueryAdaptor.managed_service_accounts_enabled?()},
+         {:ok, policy} <- get_iam_policy_for_project(project_id),
+         {:contains?, _policy, false} <-
+           {:contains?, policy, contains_managed_service_accounts?(policy)} do
+      append_managed_service_accounts(project_id, policy)
+    else
+      {:contains?, policy, _} -> {:ok, policy}
+      {:enabled?, false} -> {:error, :managed_service_accounts_disabled}
+      {:error, _err} = err -> err
+    end
+  end
+
+  defp get_iam_policy_for_project(project_id) do
+    conn = GenUtils.get_conn()
+
+    Api.Projects.cloudresourcemanager_projects_get_iam_policy(
+      conn,
+      project_id,
+      body: %Model.GetIamPolicyRequest{}
+    )
+  end
+
   # returns false if missing any of the managed service accounts
   defp contains_managed_service_accounts?(%Model.Policy{bindings: bindings}) do
     ids = BigQueryAdaptor.managed_service_account_ids()

lib/logflare/user.ex

@@ -33,6 +33,7 @@ defmodule Logflare.User do
              :bigquery_dataset_id,
              :bigquery_reservation_alerts,
              :bigquery_reservation_search,
+             :bigquery_additional_projects,
              :api_quota,
              :company,
              :token,
@@ -83,6 +84,7 @@ defmodule Logflare.User do
     field :bigquery_reservation_alerts, :string
     field :bigquery_processed_bytes_limit, :integer
     field :bigquery_enable_managed_service_accounts, :boolean, default: false
+    field :bigquery_additional_projects, :string
     field :api_quota, :integer, default: @default_user_api_quota
     field :valid_google_account, :boolean
     field :provider_uid, :string
@@ -122,6 +124,7 @@ defmodule Logflare.User do
     :bigquery_reservation_search,
     :bigquery_processed_bytes_limit,
     :bigquery_enable_managed_service_accounts,
+    :bigquery_additional_projects,
     :valid_google_account,
     :provider_uid,
     :company,
@@ -203,6 +206,16 @@ defmodule Logflare.User do
     |> validate_gcp_project(:bigquery_project_id, user_id: user.id)
   end
 
+  @spec parse_bigquery_additional_projects(t()) :: [String.t()]
+  def parse_bigquery_additional_projects(%__MODULE__{bigquery_additional_projects: nil}), do: []
+
+  def parse_bigquery_additional_projects(%__MODULE__{bigquery_additional_projects: projects}) do
+    projects
+    |> String.split(",")
+    |> Enum.map(&String.trim/1)
+    |> Enum.reject(&(&1 == ""))
+  end
+
   def hide_bigquery_defaults(user) do
     if user.bigquery_project_id == bq_project_id() do
       %{
@@ -212,7 +225,8 @@ defmodule Logflare.User do
           bigquery_dataset_location: nil,
           bigquery_processed_bytes_limit: nil,
           bigquery_reservation_alerts: nil,
-          bigquery_reservation_search: nil
+          bigquery_reservation_search: nil,
+          bigquery_additional_projects: nil
       }
     else
       user

lib/logflare_web/controllers/user_controller.ex

@@ -23,6 +23,7 @@ defmodule LogflareWeb.UserController do
     bigquery_reservation_search
     bigquery_processed_bytes_limit
     bigquery_enable_managed_service_accounts
+    bigquery_additional_projects
   )a
 
   def api_show(%{assigns: %{user: user}} = conn, _params) do
@@ -86,8 +87,11 @@ defmodule LogflareWeb.UserController do
         if updated_user.bigquery_project_id != user.bigquery_project_id,
           do: Supervisor.reset_all_user_sources(user)
 
-        if updated_user.bigquery_enable_managed_service_accounts do
-          # update iam policy
+        iam_projects_changed =
+          updated_user.bigquery_additional_projects != user.bigquery_additional_projects
+
+        if updated_user.bigquery_enable_managed_service_accounts or iam_projects_changed do
+          # update iam policy for primary and additional projects
           BigQueryAdaptor.update_iam_policy(updated_user)
         end
 

lib/logflare_web/templates/user/edit_bq_form.html.eex

@@ -45,6 +45,11 @@
       Optional. Enable routing API requests through managed service accounts for queries made to BigQuery.
       Requires the <code>resourcemanager.projects.setIamPolicy</code> permission on the project.
     </small>
+    <%= text_input e, :bigquery_additional_projects, placeholder: "project-id-1, project-id-2", class: "form-control form-control-margin", readonly: true %>
+    <%= error_tag e, :bigquery_additional_projects %>
+    <small class="form-text text-muted">
+      Optional. Comma-separated list of additional BigQuery project IDs for managed service account IAM setup.
+    </small>
   </div>
   <%= submit "Update BigQuery settings", class: "btn btn-primary form-button" %>
 <% end %>
@@ -56,6 +61,7 @@
 <%= hidden_input e, :bigquery_reservation_alerts, value: nil %>
 <%= hidden_input e, :bigquery_processed_bytes_limit, value: 10_000_000_000 %>
 <%= hidden_input e, :bigquery_enable_managed_service_accounts, value: false %>
+<%= hidden_input e, :bigquery_additional_projects, value: nil %>
 <%= submit "Reset BigQuery settings", class: "btn btn-secondary form-button" %>
 <% end %>
 <% else %>
@@ -106,6 +112,11 @@
       Optional. Enable routing API requests through managed service accounts for queries made to BigQuery.
       Requires the <code>resourcemanager.projects.setIamPolicy</code> permission on the project.
     </small>
+    <%= text_input e, :bigquery_additional_projects, placeholder: "project-id-1, project-id-2", class: "form-control form-control-margin" %>
+    <%= error_tag e, :bigquery_additional_projects %>
+    <small class="form-text text-muted">
+      Optional. Comma-separated list of additional BigQuery project IDs for managed service account IAM setup.
+    </small>
   </div>
   <%= submit "Update BigQuery settings", class: "btn btn-primary form-button" %>
 <% end %>

priv/repo/migrations/20260403000000_add_bigquery_additional_projects_to_users.exs

@@ -0,0 +1,9 @@
+defmodule Logflare.Repo.Migrations.AddBigqueryAdditionalProjectsToUsers do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      add :bigquery_additional_projects, :string
+    end
+  end
+end

priv/repo/seeds.exs

@@ -202,7 +202,8 @@ plans = [
     limit_source_fields_limit: 500,
     limit_source_ttl: 5_184_000_000,
     limit_key_values: 10_000_000,
-    type: "metered"
+    type: "metered",
+    stripe_id: "price_1Jn5crLvvReWx3Fx09NdM5ki"
   },
   %{
     name: "Metered BYOB",
@@ -217,7 +218,8 @@ plans = [
     limit_source_fields_limit: 500,
     limit_source_ttl: 5_184_000_000,
     limit_key_values: 10_000_000,
-    type: "metered"
+    type: "metered",
+    stripe_id: "price_1Jn59kLvvReWx3FxPBXNS4Me"
   }
 ]