feat: use local json logs for observability

.gitignore

@@ -85,11 +85,13 @@ pip-wheel-metadata/
 # Logs
 # ---------------------------------------------------------------------------
 *.log
+logs/
 
 # ---------------------------------------------------------------------------
 # Local runtime state (Google ADK / agent sessions, etc.)
 # ---------------------------------------------------------------------------
 **/.adk/
+.adk_state/
 
 # Local runtime data (Qdrant vector store, etc.)
 data/

compose.yaml

@@ -18,9 +18,7 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
     volumes:
-      - agent_artifacts:/app/src/.adk
+      - ./.adk_state:/app/src/.adk
+      - ./logs:/app/logs
     restart: always
     command: python -m blacki.server
-
-volumes:
-  agent_artifacts:

src/blacki/server.py

@@ -25,6 +25,7 @@
     configure_otel_resource,
     initialize_environment,
     setup_logging,
+    setup_tracing,
     validation,
 )
 
@@ -39,6 +40,7 @@
 GoogleADKInstrumentor().instrument()
 
 setup_logging(log_level=env.log_level)
+setup_tracing()
 
 _telegram_bot = None
 _container: AppContainer | None = None

src/blacki/utils/__init__.py

@@ -3,13 +3,14 @@
 from . import validation
 from .config import ServerEnv, initialize_environment
 from .exceptions import ConfigurationError
-from .observability import configure_otel_resource, setup_logging
+from .observability import configure_otel_resource, setup_logging, setup_tracing
 
 __all__ = [
     "ConfigurationError",
     "ServerEnv",
     "configure_otel_resource",
     "initialize_environment",
     "setup_logging",
+    "setup_tracing",
     "validation",
 ]

src/blacki/utils/observability.py

@@ -4,22 +4,29 @@
 OpenTelemetry environment variables for vendor-neutral operation.
 """
 
+import json
 import logging
 import os
 import sys
 import uuid
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from pathlib import Path
 
-from opentelemetry._logs import set_logger_provider
-from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
-from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
-from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry import trace
 from opentelemetry.sdk.resources import (
     SERVICE_INSTANCE_ID,
     SERVICE_NAME,
     SERVICE_NAMESPACE,
     SERVICE_VERSION,
     Resource,
 )
+from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    SpanExporter,
+    SpanExportResult,
+)
 
 
 def configure_otel_resource(agent_name: str) -> None:
@@ -41,59 +48,151 @@ def configure_otel_resource(agent_name: str) -> None:
     )
 
 
+def get_log_dir() -> Path:
+    """Get the appropriate log directory based on environment."""
+    return Path("/app/logs") if Path("/.dockerenv").exists() else Path("./logs")
+
+
+class JSONFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        log_record = {
+            "timestamp": datetime.fromtimestamp(record.created, tz=UTC).isoformat(),
+            "level": record.levelname,
+            "name": record.name,
+            "message": record.getMessage(),
+            "process_id": record.process,
+            "thread_id": record.thread,
+        }
+        if record.exc_info:
+            log_record["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_record, default=str)
+
+
 def setup_logging(log_level: str) -> None:
-    """Set up basic logging with optional OTLP log export.
+    """Set up basic logging with local JSON file export.
 
-    Configures Python logging to output to stdout. If OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-    is set, also exports logs via OTLP for correlation with traces.
+    Configures Python logging to output to stdout and append to a local JSON file.
 
     Args:
         log_level: Logging verbosity level as string
     """
     level = getattr(logging, log_level.upper(), logging.INFO)
 
-    # Configure root logger with stdout handler
-    logging.basicConfig(
-        level=level,
-        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
-        handlers=[logging.StreamHandler(sys.stdout)],
+    # Configure stdout handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setFormatter(
+        logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
     )
 
+    handlers: list[logging.Handler] = [console_handler]
+
+    # Configure local JSON file handler
+    log_dir = get_log_dir()
+    try:
+        log_dir.mkdir(parents=True, exist_ok=True)
+        file_handler = logging.FileHandler(log_dir / "blacki-telemetry.log")
+        file_handler.setFormatter(JSONFormatter())
+        handlers.append(file_handler)
+    except OSError as e:
+        print(f"⚠️ Failed to create log directory or file handler: {e}")
+        print("   Continuing with stdout logging only...")
+
+    # Configure root logger
+    logging.basicConfig(level=level, handlers=handlers, force=True)
+
     # Set levels for some noisy libraries if needed
     logging.getLogger("urllib3").setLevel(logging.WARNING)
 
-    # Set up OTLP log export if endpoint is configured
-    logs_endpoint = os.getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
-    if logs_endpoint:
+
+class JSONFileSpanExporter(SpanExporter):
+    """Exports OpenTelemetry Spans to a local JSON Lines file."""
+
+    def __init__(self, log_path: str):
+        self.log_path = log_path
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
         try:
-            print(f"📊 Configuring OTLP log export to: {logs_endpoint}")
-
-            # Create resource from OTEL_RESOURCE_ATTRIBUTES
-            resource_attrs = {}
-            if "OTEL_RESOURCE_ATTRIBUTES" in os.environ:
-                for pair in os.environ["OTEL_RESOURCE_ATTRIBUTES"].split(","):
-                    if "=" in pair:
-                        key, value = pair.split("=", 1)
-                        resource_attrs[key] = value
-
-            resource = Resource.create(resource_attrs)
-
-            # Set up logger provider with OTLP exporter
-            provider = LoggerProvider(resource=resource)
-            provider.add_log_record_processor(
-                BatchLogRecordProcessor(OTLPLogExporter())
-            )
-            set_logger_provider(provider)
-
-            # Add OTLP handler to root logger
-            handler = LoggingHandler(level=level)
-            logging.getLogger().addHandler(handler)
-
-            print("✅ OTLP log export configured")
+            with Path(self.log_path).open("a") as f:
+                for span in spans:
+                    # Convert span attributes and events to serializable dicts
+                    span_data = {
+                        "name": span.name,
+                        "context": {
+                            "trace_id": format(span.context.trace_id, "032x"),
+                            "span_id": format(span.context.span_id, "016x"),
+                        },
+                        "parent_id": format(span.parent.span_id, "016x")
+                        if span.parent
+                        else None,
+                        "kind": span.kind.name if span.kind else None,
+                        "start_time": datetime.fromtimestamp(
+                            span.start_time / 1e9, tz=UTC
+                        ).isoformat()
+                        if span.start_time
+                        else None,
+                        "end_time": datetime.fromtimestamp(
+                            span.end_time / 1e9, tz=UTC
+                        ).isoformat()
+                        if span.end_time
+                        else None,
+                        "status": {
+                            "status_code": span.status.status_code.name
+                            if span.status
+                            else None,
+                            "description": span.status.description
+                            if span.status
+                            else None,
+                        },
+                        "attributes": dict(span.attributes) if span.attributes else {},
+                        "events": [
+                            {
+                                "name": event.name,
+                                "timestamp": datetime.fromtimestamp(
+                                    event.timestamp / 1e9, tz=UTC
+                                ).isoformat()
+                                if event.timestamp
+                                else None,
+                                "attributes": dict(event.attributes)
+                                if event.attributes
+                                else {},
+                            }
+                            for event in span.events
+                        ]
+                        if span.events
+                        else [],
+                    }
+                    f.write(json.dumps(span_data, default=str) + "\n")
+            return SpanExportResult.SUCCESS
         except Exception as e:
-            print(f"⚠️ Failed to configure OTLP log export: {e}")
-            print("   Continuing with stdout logging only...")
-            logging.getLogger(__name__).warning(
-                "OTLP log export setup failed, falling back to stdout-only logging: %s",
-                e,
-            )
+            print(f"⚠️ Failed to write trace to {self.log_path}: {e}")
+            return SpanExportResult.FAILURE
+
+    def shutdown(self) -> None:
+        pass
+
+
+def setup_tracing() -> None:
+    """Set up OpenTelemetry tracing with local JSON file export."""
+    log_dir = get_log_dir()
+    log_path = log_dir / "blacki-traces.log"
+
+    try:
+        log_dir.mkdir(parents=True, exist_ok=True)
+    except OSError as e:
+        print(f"⚠️ Failed to create trace directory: {e}")
+        return
+
+    # Resource automatically reads OTEL_RESOURCE_ATTRIBUTES from environment variables
+    resource = Resource.create()
+
+    # Set up tracer provider
+    provider = TracerProvider(resource=resource)
+
+    # Add our custom JSON exporter
+    exporter = JSONFileSpanExporter(str(log_path))
+    processor = BatchSpanProcessor(exporter)
+    provider.add_span_processor(processor)
+
+    # Register global tracer provider
+    trace.set_tracer_provider(provider)
+    print(f"✅ Local trace export configured to: {log_path}")