Basic telemetry

.env

@@ -30,4 +30,9 @@ FARGATE_LOG_GROUP=/ecs/pontoon-fargate
 FARGATE_POSTGRES_HOST=localhost
 FARGATE_CELERY_BROKER_URL=redis://localhost:6379/0
 FARGATE_CELERY_RESULT_BACKEND=redis://localhost:6379/1
-FARGATE_API_ENDPOINT=http://localhost:8000
+FARGATE_API_ENDPOINT=http://localhost:8000
+
+# Posthog Telemtry
+
+POSTHOG_PAPIK=phc_G6QfHRlBV14HWfamqQqCnH4Zzm3JJ2uDMsxhTYohzZ9
+PONTOON_TELEMETRY_DISABLED=false

Dockerfile

@@ -16,6 +16,7 @@ ENV CELERY_BROKER_URL=redis://localhost:6379/0
 ENV CELERY_RESULT_BACKEND=redis://localhost:6379/1
 ENV PONTOON_API_ENDPOINT=http://localhost:8000
 ENV ALLOW_ORIGIN=http://localhost:3000
+ENV POSTHOG_PAPIK=phc_G6QfHRlBV14HWfamqQqCnH4Zzm3JJ2uDMsxhTYohzZ9
 
 
 WORKDIR /pontoon

README.md

@@ -87,20 +87,17 @@ Pontoon is used by vendors (eg. Salesforce) to **provide data syncs** as a produ
 
 # Pontoon vs. Other ETL / Reverse-ETL Platforms
 
-|                                        | **Pontoon**   | Airbyte | Singer/Stitch      | Fivetran | Hightouch                 | Prequel | Bobsled |
-| ---------------------------------      | ------------- | ------- | ------------------ | -------- | ------------------------- | ------- | ------- |
-| **Open Source**                        | ✅ Yes         | ✅ Yes   | ✅ Singer only      | ❌ No     | ⚠️ Some SDKs              | ❌ No    | ❌ No    |
-| **Self-Hosted Option**                 | ✅ Yes         | ✅ Yes   | ✅ Yes              | ❌ No     | ✅ Yes                     | ✅ Yes    | ❌ No    |
-| **First-class Data Products**          | ✅ Yes         | ❌ No    | ❌ No               | ❌ No     | ⚠️ Possible (with effort) | ✅ Yes   | ✅ Yes   |
-| **Multi-Tenant Data Export**           | ✅ Yes         | ❌ No    | ❌ No               | ❌ No     | ⚠️ Custom                 | ✅ Yes   | ✅ Yes   |
-| **Direct Data Warehouse Integrations** | ✅ Yes         | ✅ Yes   | ⚠️ Destinations only     | ⚠️ Destinations only | ⚠️ Sources only          | ✅ Yes   | ✅ Yes |
-| **DBT Integration**                   | 🚧 Coming soon  | ❌ No    | ❌ No               | ⚠️ Limited | ✅ Yes | ❌ No | ❌ No |
-| **Bulk Transfers (millions/billions of rows)** | ✅ Yes   | ✅ Yes   | ⚠️ Possible        | ⚠️ Possible | ❌ No                    | ✅ Yes   | ✅ Yes |
-| **Full Database/Table Replication**         | ❌ No          | ✅ Yes   | ❌ No               | ❌ No | ❌ No                       | ❌ No     | ❌ No |  
-| **Free to Use**                        | ✅ Yes         | ✅ Yes   | ✅ Yes (Singer CLI) | ❌ No     | ⚠️ Limited                 | ❌ No    | ❌ No    |
-
-
-
+|                                                | **Pontoon**    | Airbyte | Singer/Stitch        | Fivetran             | Hightouch                 | Prequel | Bobsled |
+| ---------------------------------------------- | -------------- | ------- | -------------------- | -------------------- | ------------------------- | ------- | ------- |
+| **Open Source**                                | ✅ Yes         | ✅ Yes  | ✅ Singer only       | ❌ No                | ⚠️ Some SDKs              | ❌ No   | ❌ No   |
+| **Self-Hosted Option**                         | ✅ Yes         | ✅ Yes  | ✅ Yes               | ❌ No                | ✅ Yes                    | ✅ Yes  | ❌ No   |
+| **First-class Data Products**                  | ✅ Yes         | ❌ No   | ❌ No                | ❌ No                | ⚠️ Possible (with effort) | ✅ Yes  | ✅ Yes  |
+| **Multi-Tenant Data Export**                   | ✅ Yes         | ❌ No   | ❌ No                | ❌ No                | ⚠️ Custom                 | ✅ Yes  | ✅ Yes  |
+| **Direct Data Warehouse Integrations**         | ✅ Yes         | ✅ Yes  | ⚠️ Destinations only | ⚠️ Destinations only | ⚠️ Sources only           | ✅ Yes  | ✅ Yes  |
+| **DBT Integration**                            | 🚧 Coming soon | ❌ No   | ❌ No                | ⚠️ Limited           | ✅ Yes                    | ❌ No   | ❌ No   |
+| **Bulk Transfers (millions/billions of rows)** | ✅ Yes         | ✅ Yes  | ⚠️ Possible          | ⚠️ Possible          | ❌ No                     | ✅ Yes  | ✅ Yes  |
+| **Full Database/Table Replication**            | ❌ No          | ✅ Yes  | ❌ No                | ❌ No                | ❌ No                     | ❌ No   | ❌ No   |
+| **Free to Use**                                | ✅ Yes         | ✅ Yes  | ✅ Yes (Singer CLI)  | ❌ No                | ⚠️ Limited                | ❌ No   | ❌ No   |
 
 # Quick Start
 
@@ -118,7 +115,8 @@ docker run \
     ghcr.io/pontoon-data/pontoon/pontoon-unified:latest
 ```
 
-> Note: If you're using CMD or Powershell, run the command in one line, without `\`
+> [!NOTE]
+> If you're using CMD or Powershell, run the command in one line, without `\`
 
 To view the Web UI: `localhost:3000`. To view the OpenAPI docs / test the API: `localhost:8000/docs`.
 
@@ -134,6 +132,9 @@ To build Pontoon from source, use Docker Compose.
 docker compose up --build
 ```
 
+> [!NOTE]
+> Pontoon collects anonymous usage data by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `PONTOON_TELEMETRY_DISABLED` environment variable to `true`. Please refer to our [telemetry docs](https://pontoon-data.github.io/Pontoon/getting-started/telemetry/) for more information.
+
 ## Creating Your First Data Export
 
 To quickly set up your first transfer in Pontoon, follow the steps in the [transfer quick start](https://pontoon-data.github.io/Pontoon/getting-started/transfer-quick-start/) guide!

api/app/config.py

@@ -23,5 +23,10 @@ class Settings(BaseSettings):
     # skip source/destination checks - always succeed 
     skip_transfers: Optional[bool] = Field(default=False)
 
+    # Telemetry
+    pontoon_telemetry_disabled: Optional[bool] = Field(default=False)
+    posthog_papik: Optional[str] = Field(default=None)
+    posthog_host: Optional[str] = Field(default="https://us.i.posthog.com")
+
     # Use this to load settings from an env file when developing locally
     # model_config  = SettingsConfigDict(env_file=f"{os.environ.get('ENV', 'dev')}.env")

api/app/dependencies.py

@@ -1,7 +1,10 @@
 from functools import lru_cache
+from typing import Dict, Any, Optional
+import uuid
 from fastapi import Depends, Security
 from fastapi.security import SecurityScopes, HTTPAuthorizationCredentials, HTTPBearer
 from sqlmodel import Field, Session, SQLModel, create_engine, select, text, func, JSON, Column
+import posthog
 
 from app.auth.json_web_token import JWTPayloadModel, JsonWebToken
 from app.auth.custom_exceptions import RequiresAuthenticationException
@@ -58,3 +61,47 @@ def get_auth():
     #     audience=settings.audience,
     #     algorithm=settings.jwt_algorithm
     # ).validate()
+
+def init_telemetry():
+    settings = get_settings()
+    if settings.pontoon_telemetry_disabled or not settings.posthog_papik:
+        print("Telemetry is disabled")
+        posthog.disabled = True
+    else:
+        print("Telemetry is enabled")
+        posthog.api_key = settings.posthog_papik
+        posthog.host = settings.posthog_host
+
+
+# Generate a persistent anonymous UUID for telemetry for this instance
+ANONYMOUS_UUID = str(uuid.uuid4())
+
+def send_telemetry_event(
+    event: str, 
+    properties: Optional[Dict[str, Any]] = None
+) -> None:
+    """
+    Send an event to PostHog if telemetry is enabled.
+    Uses a persistent anonymous UUID as distinct_id for privacy.
+
+    Args:
+        event: The event name
+        properties: Additional properties to send with the event
+    """
+    settings = get_settings()
+
+    if settings.pontoon_telemetry_disabled or not settings.posthog_papik:
+        return
+
+    try:
+        if properties is None:
+            properties = {}
+
+        # Use the persistent anonymous UUID as distinct_id
+        posthog.capture(
+            distinct_id=ANONYMOUS_UUID,
+            event=event,
+            properties=properties
+        )
+    except Exception as e:
+        print(f"Telemetry error: {e}")

api/app/main.py

@@ -6,7 +6,7 @@
 from fastapi import FastAPI, Depends, Request
 from fastapi.middleware.cors import CORSMiddleware
 
-from app.dependencies import get_settings, get_session
+from app.dependencies import get_settings, get_session, init_telemetry
 from app.routers import sources, models, recipients, destinations, transfers, internal
 
 settings = get_settings()
@@ -36,6 +36,7 @@ def format(self, record):
 
 
 app = FastAPI()
+init_telemetry()
 app.include_router(sources.router)
 app.include_router(models.router)
 app.include_router(recipients.router)

api/app/models/transfer_run.py

@@ -120,3 +120,41 @@ def update(session, transfer_run_id:uuid.UUID, transfer_run:Update) -> Model:
         session.commit()
         session.refresh(model)
         return model
+
+    @staticmethod
+    def get_transfer_row_count(session, transfer_run_id:uuid.UUID) -> int | None:
+        """
+        Get the total number of rows transferred for a specific transfer run. This only counts rows written to a destination.
+
+        Args:
+            session: Database session
+            transfer_run_id: UUID of the transfer run
+
+        Returns:
+            int | None: Total number of rows transferred, or None if no progress data available
+
+        Raises:
+            TransferRun.Exception: If transfer run is not found
+        """
+        transfer_run = TransferRun.get(session, transfer_run_id)
+
+        if transfer_run.output is None:
+            return None
+
+        progress_data = transfer_run.output.get('progress', None)
+        if not progress_data:
+            return None
+
+        total_rows = 0
+
+        # Sum up the processed rows from all progress entities
+        for entity_progress in progress_data.values():
+            if isinstance(entity_progress, dict) and 'processed' in entity_progress:
+                entity = entity_progress.get('entity')
+                # Only count rows written to a destination
+                if isinstance(entity, str) and entity.startswith('destination'):
+                    processed = entity_progress.get('processed', 0)
+                    if isinstance(processed, (int, float)):
+                        total_rows += int(processed)
+
+        return total_rows 

api/app/routers/destinations.py

@@ -6,7 +6,7 @@
 
 from app.models import Auth, Task, Destination, ScheduleModel, TransferRun, Transfer as TransferModel
 from app.routers.common import create_transfer_task, transfer_task_status
-from app.dependencies import get_session, get_settings, get_auth
+from app.dependencies import get_session, get_settings, get_auth, send_telemetry_event
 from app.config import Settings
 
 from pontoon import Mode
@@ -140,13 +140,21 @@ def list_destinations(session = Depends(get_session), auth:Auth = Security(get_a
 @router.post("", response_model=Destination.Public, status_code=status.HTTP_201_CREATED)
 def create_destination(destination:Destination.Create, session=Depends(get_session), auth:Auth = Security(get_auth)):
     try:
-        model = Destination.create(
+        created_destination = Destination.create(
             session,
             destination,
             created_by=auth.sub_uuid()
         )
 
-        return model
+        # Send telemetry event
+        send_telemetry_event(
+            "destination_created",
+            properties={
+                "vendor_type": created_destination.vendor_type
+            }
+        )
+
+        return created_destination
 
     except Destination.Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -256,7 +264,35 @@ def run_destination_transfer(destination_id:uuid.UUID, session=Depends(get_sessi
 
 @router.get("/{destination_id}/check/{task_id}", response_model=Task.Public)
 def destination_check_status(destination_id:uuid.UUID, task_id:uuid.UUID, session=Depends(get_session)):
-    return transfer_task_status(session, task_id)
+    task = transfer_task_status(session, task_id)
+
+    if task.meta.get('type') != 'destination-check':
+        raise HTTPException(status_code=400, detail="Task is not a destination check")
+
+    if task.meta.get('destination_id') != str(destination_id):
+        raise HTTPException(status_code=400, detail="Task does not belong to this destination")
+
+    destination = Destination.get(session, destination_id)
+    if task.status == 'COMPLETE' and task.output.get('success') == True:
+        try:
+            send_telemetry_event(
+                "destination_test_connection_success",
+                properties={
+                    "vendor_type": destination.vendor_type
+                }
+            )
+        except Exception as e:
+            print(f"Telemetry error in destination check status: {e}")
+
+    if task.status == 'COMPLETE' and task.output.get('success') == False:
+        send_telemetry_event(
+            "destination_test_connection_failure",
+            properties={
+                "vendor_type": destination.vendor_type
+            }
+        )
+
+    return task
 
 
 @router.delete("/{destination_id}")

api/app/routers/internal.py

@@ -7,6 +7,7 @@
 from app.routers.sources import get_source_by_id
 from app.routers.models import get_model_by_id
 from app.routers.destinations import get_destination_by_id
+from app.dependencies import send_telemetry_event
 
 
 router = APIRouter(
@@ -47,6 +48,46 @@ def create_transfer_run(transfer_run:TransferRun.Create, session=Depends(get_ses
 
 @router.put("/runs/{transfer_run_id}", response_model=TransferRun.Model)
 def update_transfer_run(transfer_run_id:uuid.UUID, transfer_run:TransferRun.Update, session=Depends(get_session)):
-    return TransferRun.update(session, transfer_run_id, transfer_run)
+    transfer_run = TransferRun.update(session, transfer_run_id, transfer_run)
+
+    if transfer_run.meta is None:
+        return transfer_run
+
+    transfer_run_type = transfer_run.meta.get('arguments', {}).get('type')
+    if transfer_run_type != "transfer" or transfer_run.status == 'RUNNING':
+        return transfer_run
+
+    # Get the number of rows transferred
+    row_count = TransferRun.get_transfer_row_count(session, transfer_run_id)
+
+    # Get the destination vendor type
+    transfer_id = transfer_run.transfer_id
+    destination_id = Transfer.get(session, transfer_id).destination_id
+    destination_vendor_type = Destination.get(session, destination_id).connection_info['vendor_type']
+
+    # Get the source vendor types from the models transferred
+    models = Destination.get(session, destination_id).models
+    source_ids = { Model.get(session, model_id).source_id for model_id in models }
+    source_vendor_types = { Source.get(session, source_id).connection_info['vendor_type'] for source_id in source_ids }
+
+    if transfer_run_type == "transfer" and transfer_run.status == 'SUCCESS':
+        send_telemetry_event(
+            "transfer_run_success",
+            properties={
+                "rows_transferred": row_count,
+                "destination_vendor_type": destination_vendor_type,
+                "source_vendor_types": source_vendor_types
+            }
+        )
+    elif transfer_run_type == "transfer" and transfer_run.status == 'FAILURE':
+        send_telemetry_event(
+            "transfer_run_failure",
+            properties={
+                "rows_transferred": row_count,
+                "destination_vendor_type": destination_vendor_type,
+                "source_vendor_types": source_vendor_types
+            }
+        )
+    return transfer_run
 
 

api/app/routers/models.py

@@ -3,7 +3,7 @@
 from typing import List, Annotated
 from fastapi import HTTPException, Depends, Query, APIRouter, Security, status
 
-from app.dependencies import get_session, get_auth
+from app.dependencies import get_session, get_auth, send_telemetry_event
 from app.models import Auth, Model
 
 router = APIRouter(
@@ -22,11 +22,18 @@ def get_model_by_id(session, model_id:uuid.UUID):
 @router.post("", response_model=Model.Public, status_code=status.HTTP_201_CREATED)
 def create_model(model:Model.Create, session=Depends(get_session), auth:Auth = Security(get_auth)):
     try:
-        return Model.create(
+        created_model = Model.create(
             session, 
             model,
             created_by=auth.sub_uuid()
         )
+
+        # Send telemetry event
+        send_telemetry_event(
+            "model_created"
+        )
+
+        return created_model
     except Model.Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
 

api/app/routers/recipients.py

@@ -4,7 +4,7 @@
 import uuid
 import time
 
-from app.dependencies import get_session, get_auth
+from app.dependencies import get_session, get_auth, send_telemetry_event
 from app.models import Auth, Recipient
 
 router = APIRouter(
@@ -22,12 +22,19 @@ def get_recipient_by_id(session, recipient_id:uuid.UUID):
 
 @router.post("", response_model=Recipient.Public, status_code=status.HTTP_201_CREATED)
 def create_recipient(recipient: Recipient.Create, session=Depends(get_session), auth:Auth = Security(get_auth)):
-    return Recipient.create(
+    created_recipient = Recipient.create(
         session,
         recipient,
         created_by=auth.sub_uuid(),
         organization_id=auth.org_uuid()
     )
+
+    # Send telemetry event
+    send_telemetry_event(
+        "recipient_created"
+    )
+
+    return created_recipient
 
 
 @router.get("", response_model=list[Recipient.Public])