Complete Lab 8

Author: SerggAidd

app_python/app.py

@@ -12,9 +12,12 @@
 import platform
 import socket
 import sys
+import time
 from datetime import datetime, timezone
 
-from flask import Flask, jsonify, request
+from flask import Flask, Response, g, jsonify, request
+from prometheus_client import (CONTENT_TYPE_LATEST, Counter, Gauge,
+                               Histogram, generate_latest)
 
 # Flask application instance
 app = Flask(__name__)
@@ -35,6 +38,27 @@
     "framework": "Flask",
 }
 
+# Prometheus metrics
+# Counter: total HTTP requests by method / endpoint / status code
+http_requests_total = Counter(
+    "http_requests_total",
+    "Total HTTP requests",
+    ["method", "endpoint", "status_code"],
+)
+
+# Histogram: request duration distribution
+http_request_duration_seconds = Histogram(
+    "http_request_duration_seconds",
+    "HTTP request duration in seconds",
+    ["method", "endpoint", "status_code"],
+)
+
+# Gauge: current number of requests being processed
+http_requests_in_progress = Gauge(
+    "http_requests_in_progress",
+    "HTTP requests currently being processed",
+)
+
 
 # JSON log formatter
 # Produces structured logs suitable for Loki / Promtail / Grafana
@@ -147,6 +171,17 @@ def request_info():
     }
 
 
+def normalized_endpoint():
+    """
+    Return a normalized endpoint for metrics labels.
+
+    Uses Flask route rule when available, fallback to raw request path.
+    """
+    if request.url_rule is not None:
+        return request.url_rule.rule
+    return request.path
+
+
 def endpoints_info():
     """
     Build an API endpoints list dynamically from Flask URL map.
@@ -201,6 +236,12 @@ def health():
     return jsonify(payload)
 
 
+@app.get("/metrics")
+def metrics():
+    """Prometheus metrics endpoint."""
+    return Response(generate_latest(), content_type=CONTENT_TYPE_LATEST)
+
+
 # Test-only endpoint to trigger HTTP 500 (uncomment to verify error handler)
 @app.get("/crash")
 def crash():
@@ -256,6 +297,14 @@ def internal_error(error):
 def log_requests():
     """Log basic request metadata before handling."""
 
+    # Skip Prometheus self-scrape from app HTTP metrics
+    g.track_metrics = request.path != "/metrics"
+
+    # Save request start time and increment in-progress gauge
+    if g.track_metrics:
+        g.request_start_time = time.perf_counter()
+        http_requests_in_progress.inc()
+
     info = request_info()
     app.logger.info(
         "request_started",
@@ -273,16 +322,44 @@ def log_response(response):
     """Log response status code after handling."""
 
     info = request_info()
-    app.logger.info(
-        "request_finished",
-        extra={
-            "method": info["method"],
-            "path": info["path"],
-            "status_code": response.status_code,
-            "client_ip": info["client_ip"],
-            "user_agent": info["user_agent"],
-        },
-    )
+
+    # Record Prometheus metrics after request is processed
+    if getattr(g, "track_metrics", False):
+        duration = time.perf_counter() - g.request_start_time
+        endpoint = normalized_endpoint()
+        status_code = str(response.status_code)
+
+        http_requests_total.labels(
+            method=request.method,
+            endpoint=endpoint,
+            status_code=status_code,
+        ).inc()
+
+        http_request_duration_seconds.labels(
+            method=request.method,
+            endpoint=endpoint,
+            status_code=status_code,
+        ).observe(duration)
+
+        http_requests_in_progress.dec()
+
+        latency_ms = round(duration * 1000, 2)
+    else:
+        latency_ms = None
+
+    extra_payload = {
+        "method": info["method"],
+        "path": info["path"],
+        "status_code": response.status_code,
+        "client_ip": info["client_ip"],
+        "user_agent": info["user_agent"],
+    }
+
+    # Add measured latency for normal app requests
+    if latency_ms is not None:
+        extra_payload["latency_ms"] = latency_ms
+
+    app.logger.info("request_finished", extra=extra_payload)
 
     return response
 

app_python/requirements.txt

@@ -13,3 +13,4 @@ requests==2.32.5
 urllib3==2.6.3
 Werkzeug==3.1.5
 zope.interface==8.2
+prometheus-client==0.23.1

monitoring/docker-compose.yml

@@ -43,6 +43,16 @@ services:
     depends_on:
       loki:
         condition: service_started
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9080 && printf \"GET /ready HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n\" >&3 && grep -q ready <&3' || exit 1"
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 15s
     deploy:
       resources:
         limits:
@@ -76,6 +86,44 @@ services:
       timeout: 5s
       retries: 5
       start_period: 20s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.50"
+          memory: 512M
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+
+  prometheus:
+    image: prom/prometheus:v3.9.0
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=15d"
+      - "--storage.tsdb.retention.size=10GB"
+    networks:
+      - logging
+    restart: unless-stopped
+    depends_on:
+      loki:
+        condition: service_started
+      grafana:
+        condition: service_started
+      app-python:
+        condition: service_started
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9090/-/healthy || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
     deploy:
       resources:
         limits:
@@ -112,7 +160,7 @@ services:
       resources:
         limits:
           cpus: "0.50"
-          memory: 512M
+          memory: 256M
         reservations:
           cpus: "0.10"
           memory: 128M
@@ -138,7 +186,7 @@ services:
       resources:
         limits:
           cpus: "0.50"
-          memory: 512M
+          memory: 256M
         reservations:
           cpus: "0.10"
           memory: 128M
@@ -150,4 +198,5 @@ networks:
 volumes:
   loki-data:
   grafana-data:
-  promtail-positions:
+  promtail-positions:
+  prometheus-data: