From bf95d9a1cb169144e524cbbbd9f1f364eb506c76 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 8 Aug 2025 16:17:53 -0700 Subject: [PATCH 1/8] first pass of telemetry with posthog --- .env | 6 +++- api/app/config.py | 5 ++++ api/app/dependencies.py | 49 +++++++++++++++++++++++++++++++++ api/app/main.py | 3 +- api/app/routers/destinations.py | 44 ++++++++++++++++++++++++++--- api/app/routers/internal.py | 17 +++++++++++- api/app/routers/models.py | 11 ++++++-- api/app/routers/recipients.py | 11 ++++++-- api/app/routers/sources.py | 40 +++++++++++++++++++++++++-- api/pyproject.toml | 1 + docker-compose.yml | 1 + 11 files changed, 174 insertions(+), 14 deletions(-) diff --git a/.env b/.env index 3acb58d..176c49e 100644 --- a/.env +++ b/.env @@ -30,4 +30,8 @@ FARGATE_LOG_GROUP=/ecs/pontoon-fargate FARGATE_POSTGRES_HOST=localhost FARGATE_CELERY_BROKER_URL=redis://localhost:6379/0 FARGATE_CELERY_RESULT_BACKEND=redis://localhost:6379/1 -FARGATE_API_ENDPOINT=http://localhost:8000 \ No newline at end of file +FARGATE_API_ENDPOINT=http://localhost:8000 + +# Posthog Telemtry + +POSTHOG_PAPIK=phc_G6QfHRlBV14HWfamqQqCnH4Zzm3JJ2uDMsxhTYohzZ9 \ No newline at end of file diff --git a/api/app/config.py b/api/app/config.py index 592e1f0..6ac6853 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -23,5 +23,10 @@ class Settings(BaseSettings): # skip source/destination checks - always succeed skip_transfers: Optional[bool] = Field(default=False) + # Telemetry + pontoon_telemetry_disabled: Optional[bool] = Field(default=False) + posthog_papik: Optional[str] = Field(default=None) + posthog_host: Optional[str] = Field(default="https://us.i.posthog.com") + # Use this to load settings from an env file when developing locally # model_config = SettingsConfigDict(env_file=f"{os.environ.get('ENV', 'dev')}.env") diff --git a/api/app/dependencies.py b/api/app/dependencies.py index efa6bff..fd7521f 100644 --- a/api/app/dependencies.py +++ b/api/app/dependencies.py @@ -1,7 +1,10 @@ from functools import lru_cache +from typing import Dict, Any, Optional +import uuid from fastapi import Depends, Security from fastapi.security import SecurityScopes, HTTPAuthorizationCredentials, HTTPBearer from sqlmodel import Field, Session, SQLModel, create_engine, select, text, func, JSON, Column +import posthog from app.auth.json_web_token import JWTPayloadModel, JsonWebToken from app.auth.custom_exceptions import RequiresAuthenticationException @@ -58,3 +61,49 @@ def get_auth(): # audience=settings.audience, # algorithm=settings.jwt_algorithm # ).validate() + +def init_telemetry(): + settings = get_settings() + if settings.pontoon_telemetry_disabled or not settings.posthog_papik: + print("Telemetry is disabled") + posthog.disabled = True + else: + print("Telemetry is enabled") + posthog.api_key = settings.posthog_papik + posthog.host = settings.posthog_host + + +# Generate a persistent anonymous UUID for telemetry for this instance +ANONYMOUS_UUID = str(uuid.uuid4()) + +def send_telemetry_event( + event: str, + properties: Optional[Dict[str, Any]] = None +) -> None: + """ + Send an event to PostHog if telemetry is enabled. + Uses a persistent anonymous UUID as distinct_id for privacy. + + Args: + event: The event name + properties: Additional properties to send with the event + """ + settings = get_settings() + + if settings.pontoon_telemetry_disabled or not settings.posthog_papik: + return + + try: + if properties is None: + properties = {} + # Remove IP address from properties + properties["$ip"] = "0.0.0.0" + + # Use the persistent anonymous UUID as distinct_id + posthog.capture( + distinct_id=ANONYMOUS_UUID, + event=event, + properties=properties + ) + except Exception as e: + print(f"Telemetry error: {e}") diff --git a/api/app/main.py b/api/app/main.py index 0813676..bc08da3 100644 --- a/api/app/main.py +++ b/api/app/main.py @@ -6,7 +6,7 @@ from fastapi import FastAPI, Depends, Request from fastapi.middleware.cors import CORSMiddleware -from app.dependencies import get_settings, get_session +from app.dependencies import get_settings, get_session, init_telemetry from app.routers import sources, models, recipients, destinations, transfers, internal settings = get_settings() @@ -36,6 +36,7 @@ def format(self, record): app = FastAPI() +init_telemetry() app.include_router(sources.router) app.include_router(models.router) app.include_router(recipients.router) diff --git a/api/app/routers/destinations.py b/api/app/routers/destinations.py index 7ed2e2a..7e02ecf 100644 --- a/api/app/routers/destinations.py +++ b/api/app/routers/destinations.py @@ -6,7 +6,7 @@ from app.models import Auth, Task, Destination, ScheduleModel, TransferRun, Transfer as TransferModel from app.routers.common import create_transfer_task, transfer_task_status -from app.dependencies import get_session, get_settings, get_auth +from app.dependencies import get_session, get_settings, get_auth, send_telemetry_event from app.config import Settings from pontoon import Mode @@ -140,13 +140,21 @@ def list_destinations(session = Depends(get_session), auth:Auth = Security(get_a @router.post("", response_model=Destination.Public, status_code=status.HTTP_201_CREATED) def create_destination(destination:Destination.Create, session=Depends(get_session), auth:Auth = Security(get_auth)): try: - model = Destination.create( + created_destination = Destination.create( session, destination, created_by=auth.sub_uuid() ) - return model + # Send telemetry event + send_telemetry_event( + "destination_created", + properties={ + "vendor_type": created_destination.vendor_type + } + ) + + return created_destination except Destination.Exception as e: raise HTTPException(status_code=400, detail=str(e)) @@ -256,7 +264,35 @@ def run_destination_transfer(destination_id:uuid.UUID, session=Depends(get_sessi @router.get("/{destination_id}/check/{task_id}", response_model=Task.Public) def destination_check_status(destination_id:uuid.UUID, task_id:uuid.UUID, session=Depends(get_session)): - return transfer_task_status(session, task_id) + task = transfer_task_status(session, task_id) + + if task.meta.get('type') != 'destination-check': + raise HTTPException(status_code=400, detail="Task is not a destination check") + + if task.meta.get('destination_id') != str(destination_id): + raise HTTPException(status_code=400, detail="Task does not belong to this destination") + + destination = Destination.get(session, destination_id) + if task.status == 'COMPLETE' and task.output.get('success') == True: + try: + send_telemetry_event( + "destination_test_connection_success", + properties={ + "vendor_type": destination.vendor_type + } + ) + except Exception as e: + print(f"Telemetry error in destination check status: {e}") + + if task.status == 'COMPLETE' and task.output.get('success') == False: + send_telemetry_event( + "destination_test_connection_failure", + properties={ + "vendor_type": destination.vendor_type + } + ) + + return task @router.delete("/{destination_id}") diff --git a/api/app/routers/internal.py b/api/app/routers/internal.py index f4dbbac..e0ae0bc 100644 --- a/api/app/routers/internal.py +++ b/api/app/routers/internal.py @@ -7,6 +7,7 @@ from app.routers.sources import get_source_by_id from app.routers.models import get_model_by_id from app.routers.destinations import get_destination_by_id +from app.dependencies import send_telemetry_event router = APIRouter( @@ -47,6 +48,20 @@ def create_transfer_run(transfer_run:TransferRun.Create, session=Depends(get_ses @router.put("/runs/{transfer_run_id}", response_model=TransferRun.Model) def update_transfer_run(transfer_run_id:uuid.UUID, transfer_run:TransferRun.Update, session=Depends(get_session)): - return TransferRun.update(session, transfer_run_id, transfer_run) + transfer_run = TransferRun.update(session, transfer_run_id, transfer_run) + + transfer_run_type = transfer_run.meta.get('arguments', {}).get('type') + if transfer_run_type == "transfer" and transfer_run.status == 'SUCCESS': + print("Sending transfer_run_success telemetry event") + send_telemetry_event( + "transfer_run_success" + ) + elif transfer_run_type == "transfer" and transfer_run.status == 'FAILURE': + print("Sending transfer_run_failure telemetry event") + send_telemetry_event( + "transfer_run_failure" + ) + print("telemetry event should have been sent") + return transfer_run diff --git a/api/app/routers/models.py b/api/app/routers/models.py index 5c8b677..ccc81ca 100644 --- a/api/app/routers/models.py +++ b/api/app/routers/models.py @@ -3,7 +3,7 @@ from typing import List, Annotated from fastapi import HTTPException, Depends, Query, APIRouter, Security, status -from app.dependencies import get_session, get_auth +from app.dependencies import get_session, get_auth, send_telemetry_event from app.models import Auth, Model router = APIRouter( @@ -22,11 +22,18 @@ def get_model_by_id(session, model_id:uuid.UUID): @router.post("", response_model=Model.Public, status_code=status.HTTP_201_CREATED) def create_model(model:Model.Create, session=Depends(get_session), auth:Auth = Security(get_auth)): try: - return Model.create( + created_model = Model.create( session, model, created_by=auth.sub_uuid() ) + + # Send telemetry event + send_telemetry_event( + "model_created" + ) + + return created_model except Model.Exception as e: raise HTTPException(status_code=400, detail=str(e)) diff --git a/api/app/routers/recipients.py b/api/app/routers/recipients.py index 8d5b294..9f50ae7 100644 --- a/api/app/routers/recipients.py +++ b/api/app/routers/recipients.py @@ -4,7 +4,7 @@ import uuid import time -from app.dependencies import get_session, get_auth +from app.dependencies import get_session, get_auth, send_telemetry_event from app.models import Auth, Recipient router = APIRouter( @@ -22,12 +22,19 @@ def get_recipient_by_id(session, recipient_id:uuid.UUID): @router.post("", response_model=Recipient.Public, status_code=status.HTTP_201_CREATED) def create_recipient(recipient: Recipient.Create, session=Depends(get_session), auth:Auth = Security(get_auth)): - return Recipient.create( + created_recipient = Recipient.create( session, recipient, created_by=auth.sub_uuid(), organization_id=auth.org_uuid() ) + + # Send telemetry event + send_telemetry_event( + "recipient_created" + ) + + return created_recipient @router.get("", response_model=list[Recipient.Public]) diff --git a/api/app/routers/sources.py b/api/app/routers/sources.py index 024529f..187c69c 100644 --- a/api/app/routers/sources.py +++ b/api/app/routers/sources.py @@ -6,7 +6,7 @@ from fastapi import HTTPException, Depends, Query, status, APIRouter, Security from sqlmodel import Session -from app.dependencies import get_session, get_settings, get_auth +from app.dependencies import get_session, get_settings, get_auth, send_telemetry_event from app.models import Auth, Task, Source from app.routers.common import create_transfer_task, transfer_task_status from pontoon.orchestration.client import Transfer, TransferException @@ -37,12 +37,22 @@ def list_sources(session:Session = Depends(get_session), auth:Auth = Security(ge @router.post("", response_model=Source.Public, status_code=status.HTTP_201_CREATED) def create_source(source:Source.Create, session:Session = Depends(get_session), auth:Auth = Security(get_auth)): try: - return Source.create( + created_source = Source.create( session, source, created_by=auth.sub_uuid(), organization_id=auth.org_uuid() ) + + # Send telemetry event + send_telemetry_event( + "source_created", + properties={ + "vendor_type": created_source.vendor_type + } + ) + + return created_source except Source.Exception as e: raise HTTPException(status_code=400, detail=str(e)) @@ -126,7 +136,31 @@ def create_source_check(source_id:uuid.UUID, session=Depends(get_session), auth: @router.get("/{source_id}/check/{task_id}", response_model=Task.Public) def source_check_status(source_id:uuid.UUID, task_id:uuid.UUID, session=Depends(get_session)): - return transfer_task_status(session, task_id) + task = transfer_task_status(session, task_id) + + if task.meta.get('type') != 'source-check': + raise HTTPException(status_code=400, detail="Task is not a source check") + + if task.meta.get('source_id') != str(source_id): + raise HTTPException(status_code=400, detail="Task does not belong to this source") + + source = Source.get(session, source_id) + if task.status == 'COMPLETE' and task.output.get('success') == True: + send_telemetry_event( + "source_test_connection_success", + properties={ + "vendor_type": source.vendor_type + } + ) + elif task.status == 'COMPLETE' and task.output.get('success') == False: + send_telemetry_event( + "source_test_connection_failure", + properties={ + "vendor_type": source.vendor_type + } + ) + + return task @router.post("/{source_id}/metadata", response_model=Task.Public) diff --git a/api/pyproject.toml b/api/pyproject.toml index 3a2991d..50eb893 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "pydantic-settings==2.7.1", "PyJWT==2.10.1", "SQLAlchemy==2.0.41", + "posthog==6.5.0" ] [tool.setuptools.packages.find] diff --git a/docker-compose.yml b/docker-compose.yml index 4fe3213..7f098cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ services: CELERY_BROKER_URL: ${CELERY_BROKER_URL} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND} PONTOON_API_ENDPOINT: ${API_ENDPOINT} + POSTHOG_PAPIK: ${POSTHOG_PAPIK} depends_on: - postgres - redis From 87af65bd34ab7cf319831513e72b2e162d1dd678 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 13 Aug 2025 13:06:16 -0700 Subject: [PATCH 2/8] udpate properties --- api/app/dependencies.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/app/dependencies.py b/api/app/dependencies.py index fd7521f..8a527c0 100644 --- a/api/app/dependencies.py +++ b/api/app/dependencies.py @@ -96,8 +96,6 @@ def send_telemetry_event( try: if properties is None: properties = {} - # Remove IP address from properties - properties["$ip"] = "0.0.0.0" # Use the persistent anonymous UUID as distinct_id posthog.capture( From 07acae4d7f0b183c113e1fb4943d3cb87f0f0782 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Aug 2025 18:00:39 -0700 Subject: [PATCH 3/8] Added telemetry for transfer runs --- .env | 3 +- Dockerfile | 1 + README.md | 31 ++++++------ api/app/models/transfer_run.py | 38 ++++++++++++++ api/app/routers/internal.py | 33 ++++++++++-- api/app/tests/test_transfer_run.py | 80 ++++++++++++++++++++++++++++++ docker-compose.fargate.yml | 2 + docker-compose.yml | 1 + 8 files changed, 168 insertions(+), 21 deletions(-) create mode 100644 api/app/tests/test_transfer_run.py diff --git a/.env b/.env index 176c49e..c4a465f 100644 --- a/.env +++ b/.env @@ -34,4 +34,5 @@ FARGATE_API_ENDPOINT=http://localhost:8000 # Posthog Telemtry -POSTHOG_PAPIK=phc_G6QfHRlBV14HWfamqQqCnH4Zzm3JJ2uDMsxhTYohzZ9 \ No newline at end of file +POSTHOG_PAPIK=phc_G6QfHRlBV14HWfamqQqCnH4Zzm3JJ2uDMsxhTYohzZ9 +PONTOON_TELEMETRY_DISABLED=false \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c209151..b444d40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ ENV CELERY_BROKER_URL=redis://localhost:6379/0 ENV CELERY_RESULT_BACKEND=redis://localhost:6379/1 ENV PONTOON_API_ENDPOINT=http://localhost:8000 ENV ALLOW_ORIGIN=http://localhost:3000 +ENV POSTHOG_PAPIK=phc_G6QfHRlBV14HWfamqQqCnH4Zzm3JJ2uDMsxhTYohzZ9 WORKDIR /pontoon diff --git a/README.md b/README.md index 49dfea1..ae46f8f 100644 --- a/README.md +++ b/README.md @@ -87,20 +87,17 @@ Pontoon is used by vendors (eg. Salesforce) to **provide data syncs** as a produ # Pontoon vs. Other ETL / Reverse-ETL Platforms -| | **Pontoon** | Airbyte | Singer/Stitch | Fivetran | Hightouch | Prequel | Bobsled | -| --------------------------------- | ------------- | ------- | ------------------ | -------- | ------------------------- | ------- | ------- | -| **Open Source** | ✅ Yes | ✅ Yes | ✅ Singer only | ❌ No | ⚠️ Some SDKs | ❌ No | ❌ No | -| **Self-Hosted Option** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ No | ✅ Yes | ✅ Yes | ❌ No | -| **First-class Data Products** | ✅ Yes | ❌ No | ❌ No | ❌ No | ⚠️ Possible (with effort) | ✅ Yes | ✅ Yes | -| **Multi-Tenant Data Export** | ✅ Yes | ❌ No | ❌ No | ❌ No | ⚠️ Custom | ✅ Yes | ✅ Yes | -| **Direct Data Warehouse Integrations** | ✅ Yes | ✅ Yes | ⚠️ Destinations only | ⚠️ Destinations only | ⚠️ Sources only | ✅ Yes | ✅ Yes | -| **DBT Integration** | 🚧 Coming soon | ❌ No | ❌ No | ⚠️ Limited | ✅ Yes | ❌ No | ❌ No | -| **Bulk Transfers (millions/billions of rows)** | ✅ Yes | ✅ Yes | ⚠️ Possible | ⚠️ Possible | ❌ No | ✅ Yes | ✅ Yes | -| **Full Database/Table Replication** | ❌ No | ✅ Yes | ❌ No | ❌ No | ❌ No | ❌ No | ❌ No | -| **Free to Use** | ✅ Yes | ✅ Yes | ✅ Yes (Singer CLI) | ❌ No | ⚠️ Limited | ❌ No | ❌ No | - - - +| | **Pontoon** | Airbyte | Singer/Stitch | Fivetran | Hightouch | Prequel | Bobsled | +| ---------------------------------------------- | -------------- | ------- | -------------------- | -------------------- | ------------------------- | ------- | ------- | +| **Open Source** | ✅ Yes | ✅ Yes | ✅ Singer only | ❌ No | ⚠️ Some SDKs | ❌ No | ❌ No | +| **Self-Hosted Option** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ No | ✅ Yes | ✅ Yes | ❌ No | +| **First-class Data Products** | ✅ Yes | ❌ No | ❌ No | ❌ No | ⚠️ Possible (with effort) | ✅ Yes | ✅ Yes | +| **Multi-Tenant Data Export** | ✅ Yes | ❌ No | ❌ No | ❌ No | ⚠️ Custom | ✅ Yes | ✅ Yes | +| **Direct Data Warehouse Integrations** | ✅ Yes | ✅ Yes | ⚠️ Destinations only | ⚠️ Destinations only | ⚠️ Sources only | ✅ Yes | ✅ Yes | +| **DBT Integration** | 🚧 Coming soon | ❌ No | ❌ No | ⚠️ Limited | ✅ Yes | ❌ No | ❌ No | +| **Bulk Transfers (millions/billions of rows)** | ✅ Yes | ✅ Yes | ⚠️ Possible | ⚠️ Possible | ❌ No | ✅ Yes | ✅ Yes | +| **Full Database/Table Replication** | ❌ No | ✅ Yes | ❌ No | ❌ No | ❌ No | ❌ No | ❌ No | +| **Free to Use** | ✅ Yes | ✅ Yes | ✅ Yes (Singer CLI) | ❌ No | ⚠️ Limited | ❌ No | ❌ No | # Quick Start @@ -118,12 +115,16 @@ docker run \ ghcr.io/pontoon-data/pontoon/pontoon-unified:latest ``` -> Note: If you're using CMD or Powershell, run the command in one line, without `\` +> [!NOTE] +> If you're using CMD or Powershell, run the command in one line, without `\` To view the Web UI: `localhost:3000`. To view the OpenAPI docs / test the API: `localhost:8000/docs`. Check out the [transfer quick start guide](https://pontoon-data.github.io/Pontoon/getting-started/transfer-quick-start/) to add your first source and destination. +> [!NOTE] +> Pontoon collects anonymous usage data by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `PONTOON_TELEMETRY_DISABLED` environment variable to `true`. + ## Running Pontoon with Docker Compose **Requirements:** [Docker Compose v2](https://docs.docker.com/compose/install/) diff --git a/api/app/models/transfer_run.py b/api/app/models/transfer_run.py index 1c6cf2c..c6ba2d1 100644 --- a/api/app/models/transfer_run.py +++ b/api/app/models/transfer_run.py @@ -120,3 +120,41 @@ def update(session, transfer_run_id:uuid.UUID, transfer_run:Update) -> Model: session.commit() session.refresh(model) return model + + @staticmethod + def get_transfer_row_count(session, transfer_run_id:uuid.UUID) -> int | None: + """ + Get the total number of rows transferred for a specific transfer run. This only counts rows written to a destination. + + Args: + session: Database session + transfer_run_id: UUID of the transfer run + + Returns: + int | None: Total number of rows transferred, or None if no progress data available + + Raises: + TransferRun.Exception: If transfer run is not found + """ + transfer_run = TransferRun.get(session, transfer_run_id) + + if transfer_run.output is None: + return None + + progress_data = transfer_run.output.get('progress', None) + if not progress_data: + return None + + total_rows = 0 + + # Sum up the processed rows from all progress entities + for entity_progress in progress_data.values(): + if isinstance(entity_progress, dict) and 'processed' in entity_progress: + entity = entity_progress.get('entity') + # Only count rows written to a destination + if isinstance(entity, str) and entity.startswith('destination'): + processed = entity_progress.get('processed', 0) + if isinstance(processed, (int, float)): + total_rows += int(processed) + + return total_rows diff --git a/api/app/routers/internal.py b/api/app/routers/internal.py index e0ae0bc..1c76d61 100644 --- a/api/app/routers/internal.py +++ b/api/app/routers/internal.py @@ -51,17 +51,40 @@ def update_transfer_run(transfer_run_id:uuid.UUID, transfer_run:TransferRun.Upda transfer_run = TransferRun.update(session, transfer_run_id, transfer_run) transfer_run_type = transfer_run.meta.get('arguments', {}).get('type') + if transfer_run_type != "transfer" or transfer_run.status == 'RUNNING': + return transfer_run + + # Get the number of rows transferred + row_count = TransferRun.get_transfer_row_count(session, transfer_run_id) + + # Get the destination vendor type + transfer_id = transfer_run.transfer_id + destination_id = Transfer.get(session, transfer_id).destination_id + destination_vendor_type = Destination.get(session, destination_id).connection_info['vendor_type'] + + # Get the source vendor types from the models transferred + models = Destination.get(session, destination_id).models + source_ids = [Model.get(session, model_id).source_id for model_id in models] + source_vendor_types = set([Source.get(session, source_id).connection_info['vendor_type'] for source_id in source_ids]) + if transfer_run_type == "transfer" and transfer_run.status == 'SUCCESS': - print("Sending transfer_run_success telemetry event") send_telemetry_event( - "transfer_run_success" + "transfer_run_success", + properties={ + "rows_transferred": row_count, + "destination_vendor_type": destination_vendor_type, + "source_vendor_types": source_vendor_types + } ) elif transfer_run_type == "transfer" and transfer_run.status == 'FAILURE': - print("Sending transfer_run_failure telemetry event") send_telemetry_event( - "transfer_run_failure" + "transfer_run_failure", + properties={ + "rows_transferred": row_count, + "destination_vendor_type": destination_vendor_type, + "source_vendor_types": source_vendor_types + } ) - print("telemetry event should have been sent") return transfer_run diff --git a/api/app/tests/test_transfer_run.py b/api/app/tests/test_transfer_run.py new file mode 100644 index 0000000..a2bd346 --- /dev/null +++ b/api/app/tests/test_transfer_run.py @@ -0,0 +1,80 @@ +import uuid +from unittest.mock import Mock, patch +import pytest +from sqlmodel import Session + +from app.models.transfer_run import TransferRun +from app.models.transfer import Transfer + + +class TestTransferRun: + """Test cases for TransferRun model""" + + def test_get_transfer_row_count_normal_operation(self): + """Test get_transfer_row_count with various valid progress data scenarios""" + mock_session = Mock() + + # Test 1: Mixed source and destination operations + mock_transfer_run_1 = Mock() + mock_transfer_run_1.output = { + 'progress': { + 'source_1': { + 'entity': 'source_1', + 'processed': 1000, + 'total': 1000, + 'percent': 100.0 + }, + 'destination_1': { + 'entity': 'destination_1', + 'processed': 1000, + 'total': 1000, + 'percent': 100.0 + } + } + } + + with patch.object(TransferRun, 'get', return_value=mock_transfer_run_1): + result = TransferRun.get_transfer_row_count(mock_session, uuid.uuid4()) + assert result == 1000 # Only destination rows counted + + def test_get_transfer_row_count_zero_rows_transferred(self): + mock_session = Mock() + mock_transfer_run_3 = Mock() + mock_transfer_run_3.output = { + 'progress': { + 'source_1': {'entity': 'source_1', 'processed': 100}, + 'destination_1': {'entity': 'destination_1', 'processed': 0} + } + } + + with patch.object(TransferRun, 'get', return_value=mock_transfer_run_3): + result = TransferRun.get_transfer_row_count(mock_session, uuid.uuid4()) + assert result == 0 # 0 destination rows + + def test_get_transfer_row_count_edge_cases(self): + """Test get_transfer_row_count with edge cases and malformed data - returns None only when no progress data exists""" + mock_session = Mock() + + # Test 1: No output field + mock_transfer_run_1 = Mock() + mock_transfer_run_1.output = None + + with patch.object(TransferRun, 'get', return_value=mock_transfer_run_1): + result = TransferRun.get_transfer_row_count(mock_session, uuid.uuid4()) + assert result is None + + # Test 2: No progress field + mock_transfer_run_2 = Mock() + mock_transfer_run_2.output = {'some_other_field': 'value'} + + with patch.object(TransferRun, 'get', return_value=mock_transfer_run_2): + result = TransferRun.get_transfer_row_count(mock_session, uuid.uuid4()) + assert result is None + + # Test 3: Empty progress field + mock_transfer_run_3 = Mock() + mock_transfer_run_3.output = {'some_other_field': 'value', 'progress': {}} + + with patch.object(TransferRun, 'get', return_value=mock_transfer_run_3): + result = TransferRun.get_transfer_row_count(mock_session, uuid.uuid4()) + assert result is None \ No newline at end of file diff --git a/docker-compose.fargate.yml b/docker-compose.fargate.yml index 481e1b2..b480e37 100644 --- a/docker-compose.fargate.yml +++ b/docker-compose.fargate.yml @@ -30,6 +30,8 @@ services: CELERY_BROKER_URL: ${FARGATE_CELERY_BROKER_URL} CELERY_RESULT_BACKEND: ${FARGATE_CELERY_RESULT_BACKEND} PONTOON_API_ENDPOINT: ${FARGATE_API_ENDPOINT} + POSTHOG_PAPIK: ${POSTHOG_PAPIK} + PONTOON_TELEMETRY_DISABLED: ${PONTOON_TELEMETRY_DISABLED} logging: driver: awslogs options: diff --git a/docker-compose.yml b/docker-compose.yml index 7f098cb..d72c045 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND} PONTOON_API_ENDPOINT: ${API_ENDPOINT} POSTHOG_PAPIK: ${POSTHOG_PAPIK} + PONTOON_TELEMETRY_DISABLED: ${PONTOON_TELEMETRY_DISABLED} depends_on: - postgres - redis From 757a85abd9dfcd5ceb3689cc2932a140b2fed37e Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Aug 2025 18:07:18 -0700 Subject: [PATCH 4/8] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ae46f8f..7988925 100644 --- a/README.md +++ b/README.md @@ -122,9 +122,6 @@ To view the Web UI: `localhost:3000`. To view the OpenAPI docs / test the API: ` Check out the [transfer quick start guide](https://pontoon-data.github.io/Pontoon/getting-started/transfer-quick-start/) to add your first source and destination. -> [!NOTE] -> Pontoon collects anonymous usage data by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `PONTOON_TELEMETRY_DISABLED` environment variable to `true`. - ## Running Pontoon with Docker Compose **Requirements:** [Docker Compose v2](https://docs.docker.com/compose/install/) @@ -135,6 +132,9 @@ To build Pontoon from source, use Docker Compose. docker compose up --build ``` +> [!NOTE] +> Pontoon collects anonymous usage data by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `PONTOON_TELEMETRY_DISABLED` environment variable to `true`. + ## Creating Your First Data Export To quickly set up your first transfer in Pontoon, follow the steps in the [transfer quick start](https://pontoon-data.github.io/Pontoon/getting-started/transfer-quick-start/) guide! From 80e69ec570d6bb757ba3fa266edcbbb8fe28b470 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Aug 2025 22:23:46 -0700 Subject: [PATCH 5/8] Add telemetry section to the readme and added a docs page --- README.md | 2 +- docs/docs/getting-started/telemetry.md | 14 ++++++++++++++ docs/mkdocs.yml | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 docs/docs/getting-started/telemetry.md diff --git a/README.md b/README.md index 7988925..3cea1bd 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ docker compose up --build ``` > [!NOTE] -> Pontoon collects anonymous usage data by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `PONTOON_TELEMETRY_DISABLED` environment variable to `true`. +> Pontoon collects anonymous usage data by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `PONTOON_TELEMETRY_DISABLED` environment variable to `true`. Please refer to our [telemetry docs](https://pontoon-data.github.io/Pontoon/getting-started/telemetry/) for more information. ## Creating Your First Data Export diff --git a/docs/docs/getting-started/telemetry.md b/docs/docs/getting-started/telemetry.md new file mode 100644 index 0000000..b078ee9 --- /dev/null +++ b/docs/docs/getting-started/telemetry.md @@ -0,0 +1,14 @@ +By default, Pontoon collects anonymized usage data through [PostHog](https://posthog.com/) to help us improve the performance and reliability of our tool. The data we collect includes general usage statistics and metadata such as transfer performance (e.g. error rates) to monitor the application’s health and functionality. + +If you’d like to disable all telemetry, you can do so by setting the environment variable `PONTOON_TELEMETRY_DISABLED` to `true`: + +```bash +docker run \ + -e PONTOON_TELEMETRY_DISABLED=true \ + /* additional args */ \ + ghcr.io/pontoon-data/pontoon/pontoon-unified:latest +``` + +If you're running Pontoon with Docker Compose, set `PONTOON_TELEMETRY_DISABLED=true` in your `.env` file. + +If you disabled telemetry correctly, you'll see the following log when starting Pontoon: `Telemetry is disabled` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index d5251ca..1bbf348 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -86,6 +86,7 @@ nav: - Transfer Quick Start: getting-started/transfer-quick-start.md - Other: - Architecture: getting-started/architecture.md + - Telemetry: getting-started/telemetry.md - Sources & Destinations: - Overview: sources-destinations/overview.md - Sources: From c6e62e4e1667f4c57d617dd99c2982770d759629 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Aug 2025 22:34:47 -0700 Subject: [PATCH 6/8] Updated telem --- api/app/routers/internal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/app/routers/internal.py b/api/app/routers/internal.py index 1c76d61..ae05567 100644 --- a/api/app/routers/internal.py +++ b/api/app/routers/internal.py @@ -64,8 +64,8 @@ def update_transfer_run(transfer_run_id:uuid.UUID, transfer_run:TransferRun.Upda # Get the source vendor types from the models transferred models = Destination.get(session, destination_id).models - source_ids = [Model.get(session, model_id).source_id for model_id in models] - source_vendor_types = set([Source.get(session, source_id).connection_info['vendor_type'] for source_id in source_ids]) + source_ids = { Model.get(session, model_id).source_id for model_id in models } + source_vendor_types = { Source.get(session, source_id).connection_info['vendor_type'] for source_id in source_ids } if transfer_run_type == "transfer" and transfer_run.status == 'SUCCESS': send_telemetry_event( From d03a0ebd88db2e0b7c92e4025aebf9c3c104fa1f Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Aug 2025 22:43:30 -0700 Subject: [PATCH 7/8] Fix unit test --- api/app/routers/internal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/app/routers/internal.py b/api/app/routers/internal.py index ae05567..9382e0f 100644 --- a/api/app/routers/internal.py +++ b/api/app/routers/internal.py @@ -49,6 +49,9 @@ def create_transfer_run(transfer_run:TransferRun.Create, session=Depends(get_ses @router.put("/runs/{transfer_run_id}", response_model=TransferRun.Model) def update_transfer_run(transfer_run_id:uuid.UUID, transfer_run:TransferRun.Update, session=Depends(get_session)): transfer_run = TransferRun.update(session, transfer_run_id, transfer_run) + + if transfer_run.meta is None: + return transfer_run transfer_run_type = transfer_run.meta.get('arguments', {}).get('type') if transfer_run_type != "transfer" or transfer_run.status == 'RUNNING': From 3afa6fd714d4677348f1fa2b1cde5930b3b9aa74 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Aug 2025 22:51:18 -0700 Subject: [PATCH 8/8] update transfer test --- api/app/tests/test_internal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/app/tests/test_internal.py b/api/app/tests/test_internal.py index 655b4d6..269256e 100644 --- a/api/app/tests/test_internal.py +++ b/api/app/tests/test_internal.py @@ -56,7 +56,7 @@ def test_create_transfer_run(): r = client.post(f"/internal/runs", json={ "transfer_id": transfer_id, "status": "RUNNING", - "meta": {"arguments": ["a", "b", 3]} + "meta": {"arguments": {"a": "b", "c": 3}} }) assert r.status_code == 200 @@ -74,5 +74,5 @@ def test_create_transfer_run(): o = r.json() assert o['status'] == 'SUCCESS' assert o['output'] == {"records": 1000, "time": 3} - assert o['meta'] == {"arguments": ["a", "b", 3]} + assert o['meta'] == {"arguments": {"a": "b", "c": 3}} assert o['created_at'] != None \ No newline at end of file