Lab08

Author: Linktur

.gitignore

@@ -44,3 +44,4 @@ __pycache__/
 # Monitoring
 monitoring/.env
 !monitoring/grafana/dashboards/lab07-logs-dashboard.json
+!monitoring/grafana/dashboards/lab08-metrics-dashboard.json

Lab-1/app_python/app.py

@@ -9,11 +9,43 @@
 from datetime import datetime, timezone
 
 from dotenv import load_dotenv
-from flask import Flask, g, jsonify, request
+from flask import Flask, Response, g, jsonify, request
 from flask_swagger_ui import get_swaggerui_blueprint
+from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest
 
 app = Flask(__name__)
 
+TRACKED_ENDPOINTS = {'/', '/health', '/metrics', '/swagger.json'}
+
+HTTP_REQUESTS_TOTAL = Counter(
+    'http_requests_total',
+    'Total HTTP requests',
+    ['method', 'endpoint', 'status_code']
+)
+
+HTTP_REQUEST_DURATION_SECONDS = Histogram(
+    'http_request_duration_seconds',
+    'HTTP request duration in seconds',
+    ['method', 'endpoint']
+)
+
+HTTP_REQUESTS_IN_PROGRESS = Gauge(
+    'http_requests_in_progress',
+    'HTTP requests currently being processed',
+    ['method', 'endpoint', 'status_code']
+)
+
+DEVOPS_INFO_ENDPOINT_CALLS_TOTAL = Counter(
+    'devops_info_endpoint_calls_total',
+    'Total endpoint calls for DevOps info service',
+    ['endpoint']
+)
+
+DEVOPS_INFO_SYSTEM_COLLECTION_SECONDS = Histogram(
+    'devops_info_system_collection_seconds',
+    'System info collection duration in seconds'
+)
+
 
 class JSONFormatter(logging.Formatter):
     def format(self, record: logging.LogRecord) -> str:
@@ -77,6 +109,17 @@ def get_client_ip() -> str:
     return client_ip
 
 
+def normalize_endpoint() -> str:
+    url_rule = getattr(request, 'url_rule', None)
+    endpoint = url_rule.rule if url_rule and url_rule.rule else request.path
+
+    if endpoint.startswith('/docs'):
+        return '/docs'
+    if endpoint in TRACKED_ENDPOINTS:
+        return endpoint
+    return '/other'
+
+
 # conf
 load_dotenv()
 HOST = os.getenv('HOST', '0.0.0.0')
@@ -168,7 +211,8 @@ def get_endpoints() -> list[dict]:
     """return a list of available endpoints"""
     return [
         {'path': '/', 'method': 'GET', 'description': 'Service information'},
-        {'path': '/health', 'method': 'GET', 'description': 'Health check'}
+        {'path': '/health', 'method': 'GET', 'description': 'Health check'},
+        {'path': '/metrics', 'method': 'GET', 'description': 'Prometheus metrics'},
     ]
 
 
@@ -200,6 +244,16 @@ def get_endpoints() -> list[dict]:
                     }
                 }
             }
+        },
+        '/metrics': {
+            'get': {
+                'summary': 'Prometheus metrics',
+                'responses': {
+                    '200': {
+                        'description': 'Prometheus text exposition format'
+                    }
+                }
+            }
         }
     }
 }
@@ -208,6 +262,14 @@ def get_endpoints() -> list[dict]:
 @app.before_request
 def log_request() -> None:
     g.request_started_at = time.perf_counter()
+    g.normalized_endpoint = normalize_endpoint()
+
+    HTTP_REQUESTS_IN_PROGRESS.labels(
+        method=request.method,
+        endpoint=g.normalized_endpoint,
+        status_code='in_progress',
+    ).inc()
+
     logger.info(
         'Incoming request',
         extra={
@@ -223,9 +285,32 @@ def log_request() -> None:
 @app.after_request
 def log_response(response):
     started_at = getattr(g, 'request_started_at', None)
+    duration_seconds = None
     duration_ms = None
     if started_at is not None:
-        duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+        duration_seconds = time.perf_counter() - started_at
+        duration_ms = round(duration_seconds * 1000, 2)
+
+    endpoint = getattr(g, 'normalized_endpoint', normalize_endpoint())
+    status_code = str(response.status_code)
+
+    HTTP_REQUESTS_TOTAL.labels(
+        method=request.method,
+        endpoint=endpoint,
+        status_code=status_code,
+    ).inc()
+
+    if duration_seconds is not None:
+        HTTP_REQUEST_DURATION_SECONDS.labels(
+            method=request.method,
+            endpoint=endpoint,
+        ).observe(duration_seconds)
+
+    HTTP_REQUESTS_IN_PROGRESS.labels(
+        method=request.method,
+        endpoint=endpoint,
+        status_code='in_progress',
+    ).dec()
 
     log_extra: dict[str, object] = {
         'event': 'request_end',
@@ -248,10 +333,15 @@ def log_response(response):
 @app.route('/')
 def index():
     """main endpoint"""
+    DEVOPS_INFO_ENDPOINT_CALLS_TOTAL.labels(endpoint='/').inc()
+
     uptime = get_uptime()
+    with DEVOPS_INFO_SYSTEM_COLLECTION_SECONDS.time():
+        system_info = get_system_info()
+
     payload = {
         'service': get_service_info(),
-        'system': get_system_info(),
+        'system': system_info,
         'runtime': {
             'uptime_seconds': uptime['seconds'],
             'uptime_human': uptime['human'],
@@ -267,6 +357,7 @@ def index():
 @app.route('/health')
 def health():
     """health check endpoint"""
+    DEVOPS_INFO_ENDPOINT_CALLS_TOTAL.labels(endpoint='/health').inc()
     uptime = get_uptime()
     return jsonify({
         'status': 'healthy',
@@ -275,8 +366,15 @@ def health():
     })
 
 
+@app.route('/metrics')
+def metrics():
+    DEVOPS_INFO_ENDPOINT_CALLS_TOTAL.labels(endpoint='/metrics').inc()
+    return Response(generate_latest(), mimetype=CONTENT_TYPE_LATEST)
+
+
 @app.route('/swagger.json')
 def swagger_json():
+    DEVOPS_INFO_ENDPOINT_CALLS_TOTAL.labels(endpoint='/swagger.json').inc()
     return jsonify(OPENAPI_SPEC)
 
 

Lab-1/app_python/requirements.txt

@@ -4,3 +4,5 @@ Flask==3.1.0
 flask-swagger-ui==4.11.1
 # Env support
 python-dotenv==1.0.1
+# Metrics
+prometheus-client==0.23.1

Lab-1/app_python/tests/test_app.py

@@ -72,3 +72,21 @@ def boom():
         "error": "Internal Server Error",
         "message": "An unexpected error occurred",
     }
+
+
+def test_metrics_endpoint_exposes_prometheus_metrics(client):
+    # Generate some traffic to populate metric series.
+    client.get("/")
+    client.get("/health")
+
+    response = client.get("/metrics")
+
+    assert response.status_code == 200
+    assert response.content_type.startswith("text/plain")
+
+    body = response.get_data(as_text=True)
+    assert "http_requests_total" in body
+    assert "http_request_duration_seconds_bucket" in body
+    assert "http_requests_in_progress" in body
+    assert "devops_info_endpoint_calls_total" in body
+    assert "devops_info_system_collection_seconds_bucket" in body

ansible/group_vars/all.yml.example

@@ -17,8 +17,10 @@ compose_project_dir: "/opt/{{ app_name }}"
 # Wipe safety switch (used with --tags web_app_wipe)
 web_app_wipe: false
 
-# Monitoring role (Lab07 bonus)
+# Monitoring role (Lab07-Lab08 bonus)
 monitoring_grafana_admin_user: "admin"
 monitoring_grafana_admin_password: "replace-with-strong-password"
 # monitoring_project_dir: "/opt/monitoring"
 # monitoring_app_enabled: true
+# monitoring_prometheus_retention_time: "15d"
+# monitoring_prometheus_retention_size: "10GB"

ansible/roles/monitoring/defaults/main.yml

@@ -6,11 +6,13 @@ monitoring_project_dir: /opt/monitoring
 monitoring_loki_version: "3.0.0"
 monitoring_promtail_version: "3.0.0"
 monitoring_grafana_version: "12.3.1"
+monitoring_prometheus_version: "v3.9.0"
 
 # Ports
 monitoring_loki_port: 3100
 monitoring_promtail_port: 9080
 monitoring_grafana_port: 3000
+monitoring_prometheus_port: 9090
 monitoring_app_host_port: 8000
 monitoring_app_container_port: 5000
 
@@ -19,17 +21,45 @@ monitoring_loki_schema_from: "2024-01-01"
 monitoring_loki_schema_version: "v13"
 monitoring_loki_retention_period: "168h"
 
+# Prometheus settings
+monitoring_prometheus_retention_time: "15d"
+monitoring_prometheus_retention_size: "10GB"
+monitoring_prometheus_scrape_interval: "15s"
+monitoring_prometheus_evaluation_interval: "15s"
+monitoring_prometheus_targets:
+  - job: "prometheus"
+    targets:
+      - "localhost:9090"
+  - job: "app"
+    targets:
+      - "{{ monitoring_app_name }}:{{ monitoring_app_container_port }}"
+    path: "/metrics"
+  - job: "loki"
+    targets:
+      - "loki:3100"
+    path: "/metrics"
+  - job: "grafana"
+    targets:
+      - "grafana:3000"
+    path: "/metrics"
+  - job: "promtail"
+    targets:
+      - "promtail:9080"
+    path: "/metrics"
+
 # App integration
 monitoring_app_enabled: true
 monitoring_app_name: "devops-python"
 monitoring_app_image: "{{ dockerhub_username | default('linktur') }}/devops-lab2"
 monitoring_app_tag: "{{ docker_image_tag | default('latest') }}"
 
-# Grafana security
+# Grafana security and provisioning
 monitoring_grafana_admin_user: "admin"
-monitoring_grafana_admin_password: "ChangeMe_Lab07_Replace"
-monitoring_grafana_datasource_uid: "loki"
-monitoring_grafana_dashboard_uid: "lab07-logs"
+monitoring_grafana_admin_password: "ChangeMe_Lab08_Replace"
+monitoring_grafana_loki_datasource_uid: "loki"
+monitoring_grafana_prometheus_datasource_uid: "prometheus"
+monitoring_grafana_logs_dashboard_uid: "lab07-logs"
+monitoring_grafana_metrics_dashboard_uid: "lab08-metrics"
 
 # Compose behavior
 monitoring_compose_pull_policy: "always"
@@ -48,21 +78,28 @@ monitoring_resources:
   promtail:
     limits:
       cpus: "0.5"
-      memory: "512M"
+      memory: "256M"
     reservations:
       cpus: "0.10"
       memory: "128M"
-  grafana:
+  prometheus:
     limits:
       cpus: "1.0"
       memory: "1G"
     reservations:
       cpus: "0.25"
       memory: "256M"
-  app:
+  grafana:
     limits:
       cpus: "0.5"
       memory: "512M"
+    reservations:
+      cpus: "0.25"
+      memory: "256M"
+  app:
+    limits:
+      cpus: "0.5"
+      memory: "256M"
     reservations:
       cpus: "0.10"
       memory: "128M"

ansible/roles/monitoring/tasks/deploy.yml

@@ -25,6 +25,13 @@
     delay: 2
     timeout: "{{ monitoring_wait_timeout }}"
 
+- name: Wait for Prometheus port
+  ansible.builtin.wait_for:
+    host: "127.0.0.1"
+    port: "{{ monitoring_prometheus_port }}"
+    delay: 2
+    timeout: "{{ monitoring_wait_timeout }}"
+
 - name: Wait for Grafana port
   ansible.builtin.wait_for:
     host: "127.0.0.1"
@@ -50,6 +57,16 @@
   delay: 3
   until: monitoring_loki_ready.status == 200
 
+- name: Verify Prometheus healthy endpoint
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ monitoring_prometheus_port }}/-/healthy"
+    method: GET
+    status_code: 200
+  register: monitoring_prometheus_ready
+  retries: 20
+  delay: 3
+  until: monitoring_prometheus_ready.status == 200
+
 - name: Verify Grafana health endpoint
   ansible.builtin.uri:
     url: "http://127.0.0.1:{{ monitoring_grafana_port }}/api/health"
@@ -64,7 +81,7 @@
   ansible.builtin.uri:
     url: >-
       http://127.0.0.1:{{ monitoring_grafana_port }}/api/datasources/uid/{{
-      monitoring_grafana_datasource_uid }}
+      monitoring_grafana_loki_datasource_uid }}
     method: GET
     user: "{{ monitoring_grafana_admin_user }}"
     password: "{{ monitoring_grafana_admin_password }}"
@@ -74,3 +91,18 @@
   retries: 20
   delay: 3
   until: monitoring_loki_datasource.status == 200
+
+- name: Verify Prometheus datasource in Grafana
+  ansible.builtin.uri:
+    url: >-
+      http://127.0.0.1:{{ monitoring_grafana_port }}/api/datasources/uid/{{
+      monitoring_grafana_prometheus_datasource_uid }}
+    method: GET
+    user: "{{ monitoring_grafana_admin_user }}"
+    password: "{{ monitoring_grafana_admin_password }}"
+    force_basic_auth: true
+    status_code: 200
+  register: monitoring_prometheus_datasource
+  retries: 20
+  delay: 3
+  until: monitoring_prometheus_datasource.status == 200