diff --git a/labs/app_python/Dockerfile b/labs/app_python/Dockerfile index c43ef9ed35..b1df53879a 100644 --- a/labs/app_python/Dockerfile +++ b/labs/app_python/Dockerfile @@ -4,10 +4,12 @@ WORKDIR /app RUN groupadd -r appuser && useradd -r -g appuser appuser -COPY requirements.txt . +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir -r requirements.txt + pip install --no-cache-dir -r requirements.txt && \ + pip install flask python-json-logger COPY . . diff --git a/labs/app_python/app.py b/labs/app_python/app.py index ce33da9724..a5eeb3c199 100644 --- a/labs/app_python/app.py +++ b/labs/app_python/app.py @@ -1,140 +1,132 @@ """ -DevOps Info Service -Веб-сервис для предоставления информации о системе и состоянии сервиса +DevOps Info Service with JSON Logging """ import os import socket -import platform +import logging from datetime import datetime, timezone from flask import Flask, jsonify, request +from pythonjsonlogger import jsonlogger -# Создаем приложение Flask +# Flask app app = Flask(__name__) -# Настройки из переменных окружения +# ENV config HOST = os.getenv('HOST', '0.0.0.0') PORT = int(os.getenv('PORT', 5000)) DEBUG = os.getenv('DEBUG', 'False').lower() == 'true' -# Время запуска сервиса START_TIME = datetime.now(timezone.utc) -# Добавляем Docker информацию +# Docker info IS_DOCKER = os.path.exists('/.dockerenv') CONTAINER_ID = socket.gethostname() if IS_DOCKER else None +# ---------------------------- +# JSON LOGGING CONFIG +# ---------------------------- +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +handler = logging.StreamHandler() +formatter = jsonlogger.JsonFormatter( + '%(asctime)s %(levelname)s %(message)s %(method)s %(path)s %(status)s %(client_ip)s %(duration)s' +) +handler.setFormatter(formatter) +logger.addHandler(handler) + +# ---------------------------- +# HELPERS +# ---------------------------- def get_uptime(): - """Рассчитывает время работы сервиса""" delta = datetime.now(timezone.utc) - START_TIME seconds = int(delta.total_seconds()) - - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - secs = seconds % 60 - - parts = [] - if hours > 0: - parts.append(f"{hours} hour{'s' if hours != 1 else ''}") - if minutes > 0: - parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}") - if secs > 0 or not parts: - parts.append(f"{secs} second{'s' if secs != 1 else ''}") - - return { - 'seconds': seconds, - 'human': ', '.join(parts) - } + return {'seconds': seconds, 'human': f"{seconds} seconds"} + +# ---------------------------- +# LOGGING MIDDLEWARE +# ---------------------------- +@app.before_request +def log_request(): + request.start_time = datetime.now(timezone.utc) + logging.info( + "request_received", + extra={ + "method": request.method, + "path": request.path, + "client_ip": request.remote_addr + } + ) + +@app.after_request +def log_response(response): + duration = (datetime.now(timezone.utc) - request.start_time).total_seconds() + logging.info( + "request_completed", + extra={ + "method": request.method, + "path": request.path, + "status": response.status_code, + "client_ip": request.remote_addr, + "duration": duration + } + ) + return response +# ---------------------------- +# ROUTES +# ---------------------------- @app.route('/') def main_endpoint(): - """Основной эндпоинт - информация о сервисе и системе""" - - # Получаем информацию о системе - system_info = { - 'hostname': socket.gethostname(), - 'platform': platform.system(), - 'platform_version': platform.version(), - 'architecture': platform.machine(), - 'cpu_count': os.cpu_count(), - 'python_version': platform.python_version(), - 'is_docker_container': IS_DOCKER, - 'container_id': CONTAINER_ID - } - - # Информация о времени - runtime_info = { - 'uptime_seconds': get_uptime()['seconds'], - 'uptime_human': get_uptime()['human'], - 'current_time': datetime.now(timezone.utc).isoformat(), - 'timezone': 'UTC', - 'start_time': START_TIME.isoformat() - } - - # Информация о запросе - request_info = { - 'client_ip': request.remote_addr, - 'user_agent': request.headers.get('User-Agent', 'Unknown'), - 'method': request.method, - 'path': request.path - } - return jsonify({ - 'service': { - 'name': 'devops-info-service', - 'version': '2.0.0', - 'description': 'DevOps course info service (Dockerized)', - 'framework': 'Flask', - 'environment': 'docker' if IS_DOCKER else 'local' - }, - 'system': system_info, - 'runtime': runtime_info, - 'request': request_info, - 'endpoints': [ - {'path': '/', 'method': 'GET', 'description': 'Service information'}, - {'path': '/health', 'method': 'GET', 'description': 'Health check'}, - {'path': '/docker', 'method': 'GET', 'description': 'Docker information'} - ] + 'service': 'devops-info-service', + 'status': 'running', + 'time': datetime.now(timezone.utc).isoformat() }) @app.route('/health') def health_check(): - """Эндпоинт проверки состояния сервиса""" return jsonify({ 'status': 'healthy', - 'timestamp': datetime.now(timezone.utc).isoformat(), - 'uptime_seconds': get_uptime()['seconds'], - 'environment': 'docker' if IS_DOCKER else 'local', - 'container_id': CONTAINER_ID + 'uptime': get_uptime()['seconds'] }) @app.route('/docker') def docker_info(): - """Эндпоинт информации о Docker окружении""" return jsonify({ 'is_docker': IS_DOCKER, - 'container_id': CONTAINER_ID, - 'docker_env': dict(os.environ) if IS_DOCKER else None, - 'message': 'Running in Docker container' if IS_DOCKER else 'Running locally' + 'container_id': CONTAINER_ID }) +# ---------------------------- +# ERRORS +# ---------------------------- @app.errorhandler(404) def not_found(error): - """Обработка ошибки 404""" - return jsonify({ - 'error': 'Not Found', - 'message': 'Endpoint does not exist', - 'available_endpoints': [ - {'path': '/', 'method': 'GET'}, - {'path': '/health', 'method': 'GET'}, - {'path': '/docker', 'method': 'GET'} - ] - }), 404 + logging.error( + "not_found", + extra={ + "method": request.method, + "path": request.path, + "status": 404, + "client_ip": request.remote_addr + } + ) + return jsonify({'error': 'Not Found'}), 404 +# ---------------------------- +# START +# ---------------------------- if __name__ == '__main__': - print(f"Starting DevOps Info Service on {HOST}:{PORT}") - print(f"Debug mode: {DEBUG}") - print(f"Docker environment: {IS_DOCKER}") - print(f"Container ID: {CONTAINER_ID}") - + logging.info( + "service_started", + extra={ + "host": HOST, + "port": PORT, + "debug": DEBUG, + "docker": IS_DOCKER, + "container_id": CONTAINER_ID + } + ) app.run(host=HOST, port=PORT, debug=DEBUG) \ No newline at end of file diff --git a/labs/monitoring/docker-compose.yml b/labs/monitoring/docker-compose.yml new file mode 100644 index 0000000000..d5efe8db44 --- /dev/null +++ b/labs/monitoring/docker-compose.yml @@ -0,0 +1,96 @@ +services: + loki: + image: grafana/loki:3.0.0 + container_name: loki + ports: + - "3100:3100" + command: -config.file=/etc/loki/config.yml + volumes: + - ./loki/config.yml:/etc/loki/config.yml + - loki-data:/loki + networks: + - logging + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://localhost:3100/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + promtail: + image: grafana/promtail:3.0.0 + container_name: promtail + command: -config.file=/etc/promtail/config.yml + volumes: + - ./promtail/config.yml:/etc/promtail/config.yml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "9080:9080" + networks: + - logging + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + + grafana: + image: grafana/grafana:12.3.1 + container_name: grafana + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + environment: + - GF_AUTH_ANONYMOUS_ENABLED=false + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin123 + networks: + - logging + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + + app-python: + build: ../app_python + container_name: app-python + ports: + - "5000:5000" + networks: + - logging + labels: + logging: "promtail" + app: "devops-python" + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:5000/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + loki-data: + grafana-data: + +networks: + logging: \ No newline at end of file diff --git a/labs/monitoring/docs/LAB07.md b/labs/monitoring/docs/LAB07.md new file mode 100644 index 0000000000..763a817771 --- /dev/null +++ b/labs/monitoring/docs/LAB07.md @@ -0,0 +1,207 @@ +# LAB07 — Observability & Logging with Loki Stack + +## 1. Architecture + +**Components:** + +- **Loki 3.0** — log storage (TSDB backend) +- **Promtail 3.0** — log collector, scrapes Docker container logs +- **Grafana 12.3** — visualization and dashboards +- **app-python** — Flask application with JSON logging + +**Diagram:** + +``` + +app-python ── stdout/logs ──► Promtail ──► Loki ──► Grafana + +``` + +**Network:** all services share `logging` network. + +--- + +## 2. Setup Guide + +### Prerequisites + +- Docker Desktop (WSL2 backend on Windows 11) +- Python 3.11+ +- VS Code + +### Steps + +1. Project structure: + +``` + +labs/ +├── app_python/ # Flask app + Dockerfile +└── monitoring/ # docker-compose.yml + Loki/Promtail configs + +```` + +2. Build and start stack: + +```bash +cd labs/monitoring +docker compose up -d --build +```` + +3. Verify services: + +```bash +docker compose ps +``` + +4. Access Grafana: + +``` +http://localhost:3000 +``` + +Admin credentials: `admin` / `admin123` + +--- + +## 3. Configuration + +### docker-compose.yml + +* **Volumes:** + + * `loki-data` — persistent Loki storage + * `grafana-data` — Grafana dashboards and configs +* **Ports:** + + * Loki: 3100 + * Promtail: 9080 + * Grafana: 3000 + * app-python: 5000 +* **Resource limits** configured under `deploy.resources` +* **Healthchecks** configured for all services + +### Loki config (`loki/config.yml`) + +* TSDB storage with filesystem +* Schema v13 +* Retention period: 7 days (`168h`) +* Compactor enabled + +### Promtail config (`promtail/config.yml`) + +* Docker service discovery enabled via `/var/run/docker.sock` +* Relabeling: container name → `container`, label `app` → `app` +* Clients: `http://loki:3100/loki/api/v1/push` + +--- + +## 4. Application Logging + +### app-python + +* Flask application with JSON logging using `python-json-logger` +* Logs every request and response in JSON: + +```json +{ + "asctime": "2026-03-23T07:00:00", + "levelname": "INFO", + "message": "request_completed", + "method": "GET", + "path": "/", + "status": 200, + "client_ip": "172.17.0.1", + "duration": 0.002 +} +``` + +* Error handling logs 404 and other errors +* Logging middleware uses `before_request` and `after_request` + +--- + +## 5. Dashboard + +**Dashboard:** `DevOps Logs Dashboard` +**Panels:** + +1. **All Logs** — Logs, `{app=~"devops-.*"}` +2. **Requests per second** — Time series, `sum by (app) (rate({app=~"devops-.*"}[1m]))` +3. **Error Logs** — Logs, `{app=~"devops-.*"} | json | status=404` +4. **Log Levels** — Pie / Stat, `sum by (levelname) (count_over_time({app="devops-python"} | json | __error__="" [5m]))` + +--- + +## 6. Production Config + +* Grafana security: anonymous access disabled, admin password set +* Resource limits: CPUs and memory configured in `docker-compose.yml` +* Healthchecks: defined for Loki, Promtail, Grafana, and app-python +* Volumes: persistent Loki and Grafana data + +--- + +## 7. Testing + +### Verify logs: + +```bash +curl http://localhost:5000/ +curl http://localhost:5000/health +curl http://localhost:5000/invalid # to generate 404 +``` + +### Grafana queries: + +```logql +{app="devops-python"} +{app="devops-python"} | json +{app="devops-python"} | json | method="GET" +{app="devops-python"} | json | status=404 +``` + +### Verify health: + +```bash +docker compose ps +``` + +All services should show `healthy`. + +--- + +## 8. Challenges + +* Promtail initially did not parse `app` label — fixed by `__meta_docker_container_label_app` relabel +* JSONParserErr in Log Levels panel — fixed with `| __error__=""` filter +* Grafana anonymous access needed to be disabled for production compliance + +--- + +## 9. Evidence + +### 9.1 Logs from all containers + +* Grafana Explore query: `{job="docker"}` + ![All Docker logs](../screenshots/all_logs.png) + +### 9.2 JSON logs from app-python + +* Grafana Explore query: `{app="devops-python"}` + ![App Python JSON logs](../screenshots/app_logs.png) + +### 9.3 Dashboard + +* DevOps Logs Dashboard with 4 panels: + + 1. All Logs + 2. Requests per second + 3. Error Logs + 4. Log Levels + ![DevOps Logs Dashboard](../screenshots/dashboard.png) + +### 9.4 Docker Compose status / Healthchecks + +* PowerShell command: `docker compose ps` + ![Docker Compose ps](../screenshots/healthchecks.png) diff --git a/labs/monitoring/loki/config.yml b/labs/monitoring/loki/config.yml new file mode 100644 index 0000000000..85121061c4 --- /dev/null +++ b/labs/monitoring/loki/config.yml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 168h + +compactor: + working_directory: /loki/compactor + delete_request_store: filesystem \ No newline at end of file diff --git a/labs/monitoring/promtail/config.yml b/labs/monitoring/promtail/config.yml new file mode 100644 index 0000000000..32c1460baf --- /dev/null +++ b/labs/monitoring/promtail/config.yml @@ -0,0 +1,27 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + target_label: 'container' + regex: '/(.*)' + replacement: '$1' + + - source_labels: ['__meta_docker_container_label_app'] + target_label: 'app' + + - target_label: job + replacement: docker \ No newline at end of file diff --git a/labs/monitoring/screenshots/all_logs.png b/labs/monitoring/screenshots/all_logs.png new file mode 100644 index 0000000000..0a22fe8b47 Binary files /dev/null and b/labs/monitoring/screenshots/all_logs.png differ diff --git a/labs/monitoring/screenshots/app_logs.png b/labs/monitoring/screenshots/app_logs.png new file mode 100644 index 0000000000..49ec402584 Binary files /dev/null and b/labs/monitoring/screenshots/app_logs.png differ diff --git a/labs/monitoring/screenshots/dashboard.png b/labs/monitoring/screenshots/dashboard.png new file mode 100644 index 0000000000..512f81b335 Binary files /dev/null and b/labs/monitoring/screenshots/dashboard.png differ diff --git a/labs/monitoring/screenshots/healthchecks.png b/labs/monitoring/screenshots/healthchecks.png new file mode 100644 index 0000000000..69880bfb82 Binary files /dev/null and b/labs/monitoring/screenshots/healthchecks.png differ