diff --git a/.github/workflows/ansible-deploy.yml b/.github/workflows/ansible-deploy.yml new file mode 100644 index 0000000000..ff7e9e04df --- /dev/null +++ b/.github/workflows/ansible-deploy.yml @@ -0,0 +1,79 @@ +name: Ansible Deployment + +on: + push: + branches: [main, master, lab06] + paths: + - "ansible/**" + - ".github/workflows/ansible-deploy.yml" + pull_request: + branches: [main, master, lab06] + paths: + - "ansible/**" + +jobs: + lint: + name: Ansible Lint + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install ansible ansible-lint + + - name: Run ansible-lint + env: + ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + run: | + cd ansible + echo "$ANSIBLE_VAULT_PASSWORD" > /tmp/vault_pass + export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/vault_pass + ansible-lint playbooks/*.yml + rm /tmp/vault_pass + + deploy: + name: Deploy Application + needs: lint + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Ansible + run: pip install ansible + + - name: Setup SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + - name: Deploy with Ansible + env: + ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + ANSIBLE_HOST_KEY_CHECKING: False + run: | + cd ansible + echo "$ANSIBLE_VAULT_PASSWORD" > /tmp/vault_pass + ansible-playbook playbooks/deploy.yml \ + -i inventory/hosts.ini \ + --vault-password-file /tmp/vault_pass + rm /tmp/vault_pass + + - name: Verify Deployment + run: | + sleep 10 + curl -f http://${{ secrets.VM_HOST }}:5000 || exit 1 + curl -f http://${{ secrets.VM_HOST }}:5000/health || exit 1 diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000000..8529db24df --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,67 @@ +name: Python CI + +on: + push: + paths: + - "app_python/**" + - ".github/workflows/python-ci.yml" + pull_request: + paths: + - "app_python/**" + +jobs: + test-and-build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Cache pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('app_python/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r app_python/requirements.txt + + - name: Run tests + run: | + cd app_python + pytest + + - name: Docker login + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker image version + run: echo "VERSION=$(date +%Y.%m.%d)" >> $GITHUB_ENV + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: app_python + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/devops-info-python:${{ env.VERSION }} + ${{ secrets.DOCKERHUB_USERNAME }}/devops-info-python:latest + + - name: Install Snyk + run: npm install -g snyk + + - name: Run Snyk test + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + run: snyk test --all-projects --severity-threshold=high diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..1a1486e139 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +test +minikube-darwin-arm64 +*.dmg +*.pkg +minikube* \ No newline at end of file diff --git a/README.md b/README.md index 0b159ed716..371d51f456 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,271 @@ -# DevOps Engineering Labs +# DevOps Engineering: Core Practices -## Introduction +[![Labs](https://img.shields.io/badge/Labs-18-blue)](#labs) +[![Exam](https://img.shields.io/badge/Exam-Optional-green)](#exam-alternative) +[![Duration](https://img.shields.io/badge/Duration-18%20Weeks-lightgrey)](#course-roadmap) -Welcome to the DevOps Engineering course labs! These hands-on labs are designed to guide you through various aspects of DevOps practices and principles. As you progress through the labs, you'll gain practical experience in application development, containerization, testing, infrastructure setup, CI/CD processes, and more. +Master **production-grade DevOps practices** through hands-on labs. Build, containerize, deploy, monitor, and scale applications using industry-standard tools. -## Lab Syllabus +--- -Lab 1: Web Application Development -Lab 2: Containerization -Lab 3: Continuous Integration -Lab 4: Infrastructure as Code & Terraform -Lab 5: Configuration Management -Lab 6: Ansible Automation -Lab 7: Observability, Logging, Loki Stack -Lab 8: Monitoring & Prometheus -Lab 9: Kubernetes & Declarative Manifests -Lab 10: Helm Charts & Library Charts -Lab 11: Kubernetes Secrets Management (Vault, ConfigMaps) -Lab 12: Kubernetes ConfigMaps & Environment Variables -Lab 13: GitOps with ArgoCD -Lab 14: StatefulSet Optimization -Lab 15: Kubernetes Monitoring & Init Containers -Lab 16: IPFS & Fleek Decentralization +## Quick Start -## Architecture +1. **Fork** this repository +2. **Clone** your fork locally +3. **Start with Lab 1** and progress sequentially +4. **Submit PRs** for each lab (details below) -This repository has a master branch containing an introduction. Each new lab assignment will be added as a markdown file with a lab number. +--- -## Rules +## Course Roadmap -To successfully complete the labs and pass the course, follow these rules: +| Week | Lab | Topic | Key Technologies | +|------|-----|-------|------------------| +| 1 | 1 | Web Application Development | Python/Go, Best Practices | +| 2 | 2 | Containerization | Docker, Multi-stage Builds | +| 3 | 3 | Continuous Integration | GitHub Actions, Snyk | +| 4 | 4 | Infrastructure as Code | Terraform, Cloud Providers | +| 5 | 5 | Configuration Management | Ansible Basics | +| 6 | 6 | Continuous Deployment | Ansible Advanced | +| 7 | 7 | Logging | Promtail, Loki, Grafana | +| 8 | 8 | Monitoring | Prometheus, Grafana | +| 9 | 9 | Kubernetes Basics | Minikube, Deployments, Services | +| 10 | 10 | Helm Charts | Templating, Hooks | +| 11 | 11 | Secrets Management | K8s Secrets, HashiCorp Vault | +| 12 | 12 | Configuration & Storage | ConfigMaps, PVCs | +| 13 | 13 | GitOps | ArgoCD | +| 14 | 14 | Progressive Delivery | Argo Rollouts | +| 15 | 15 | StatefulSets | Persistent Storage, Headless Services | +| 16 | 16 | Cluster Monitoring | Kube-Prometheus, Init Containers | +| — | **Exam Alternative Labs** | | | +| 17 | 17 | Edge Deployment | Fly.io, Global Distribution | +| 18 | 18 | Decentralized Storage | 4EVERLAND, IPFS, Web3 | -1. **Lab Dependency:** Complete the labs in order; each lab builds upon the previous one. -2. **Submission and Grading:** Submit your solutions as pull requests (PRs) to the master branch of this repository. You need at least 6/10 points for each lab to pass. -3. **Fork Repository:** Fork this repository to your workspace to create your own version for solving the labs. -4. **Recommended Workflow:** Build your solutions incrementally. Complete lab N based on lab N-1. -5. **PR Creation:** Create a PR from your fork to the master branch of this repository and from your fork's branch to your fork's master branch. -6. **Wait for Grade:** Once your PR is created, wait for your lab to be reviewed and graded. +--- -### Example for the first lab +## Grading -1. Fork this repository. -2. Checkout to the lab1 branch. -3. Complete the lab1 tasks. -4. Push the code to your repository. -5. Create a PR to the master branch of this repository from your fork's lab1 branch. -6. Create a PR to the master branch of your repository from your lab1 branch. -7. Wait for your grade. +### Grade Composition -## Grading and Grades Distribution +| Component | Weight | Points | +|-----------|--------|--------| +| **Labs (16 required)** | 80% | 160 pts | +| **Final Exam** | 20% | 40 pts | +| **Bonus Tasks** | Extra | +40 pts max | +| **Total** | 100% | 200 pts | -Your final grade will be determined based on labs and a final exam: +### Exam Alternative -- Labs: 70% of your final grade. -- Final Exam: 30% of your final grade. +Don't want to take the exam? Complete **both** bonus labs: -Grade ranges: +| Lab | Topic | Points | +|-----|-------|--------| +| **Lab 17** | Fly.io Edge Deployment | 20 pts | +| **Lab 18** | 4EVERLAND & IPFS | 20 pts | -- [90-100] - A -- [75-90) - B -- [60-75) - C -- [0-60) - D +**Requirements:** +- Complete both labs (17 + 18 = 40 pts, replaces exam) +- Minimum 16/20 on each lab +- Deadline: **1 week before exam date** +- Can still take exam if you need more points for desired grade -### Labs Grading +
+📊 Grade Scale -Each lab is worth 10 points. Completing main tasks correctly earns you 10 points. Completing bonus tasks correctly adds 2.5 points. You can earn a maximum of 12.5 points per lab by completing all main and bonus tasks. +| Grade | Points | Percentage | +|-------|--------|------------| +| **A** | 180-200+ | 90-100% | +| **B** | 150-179 | 75-89% | +| **C** | 120-149 | 60-74% | +| **D** | 0-119 | 0-59% | -Finishing all bonus tasks lets you skip the exam and grants you 5 extra points. Incomplete bonus tasks require you to take the exam, which could save you from failing it. +**Minimum to Pass:** 120 points (60%) ->The labs account for 70% of your final grade. With 14 labs in total, each lab contributes 5% to your final grade. Completing all main tasks in a lab earns you the maximum 10 points, which corresponds to 5% of your final grade. ->If you successfully complete all bonus tasks, you'll earn an additional 2.5 points, totaling 12.5 points for that lab, or 6.25% of your final grade. Over the course of all 14 labs, the cumulative points from bonus tasks add up to 87.5% of your final grade. ->Additionally, a 5% bonus is granted for successfully finishing all bonus tasks, ensuring that if you successfully complete everything, your final grade will be 92.5%, which corresponds to an A grade. +
-## Deadlines and Labs Distribution +
+📈 Grade Examples -Each week, two new labs will be available. You'll have one week to submit your solutions. Refer to Moodle for presentation slides and deadlines. +**Scenario 1: Labs + Exam** +``` +Labs: 16 × 9 = 144 pts +Bonus: 5 labs × 2.5 = 12.5 pts +Exam: 35/40 pts +Total: 191.5 pts = 96% (A) +``` -## Submission Policy +**Scenario 2: Labs + Exam Alternative** +``` +Labs: 16 × 9 = 144 pts +Bonus: 8 labs × 2.5 = 20 pts +Lab 17: 18 pts +Lab 18: 17 pts +Total: 199 pts = 99.5% (A) +``` -Submitting your lab results on time is crucial for your grading. Late submissions receive a maximum score of 6 points for the corresponding lab. Remember, completing all labs is necessary to successfully pass the course. +
+ +--- + +## Lab Structure + +Each lab is worth **10 points** (main tasks) + **2.5 points** (bonus). + +- **Minimum passing score:** 6/10 per lab +- **Late submissions:** Max 6/10 (within 1 week) +- **Very late (>1 week):** Not accepted + +
+📋 Lab Categories + +**Foundation (Labs 1-2)** +- Web app development +- Docker containerization + +**CI/CD & Infrastructure (Labs 3-4)** +- GitHub Actions +- Terraform + +**Configuration Management (Labs 5-6)** +- Ansible playbooks and roles + +**Observability (Labs 7-8)** +- Loki logging stack +- Prometheus monitoring + +**Kubernetes Core (Labs 9-12)** +- K8s basics, Helm +- Secrets, ConfigMaps + +**Advanced Kubernetes (Labs 13-16)** +- ArgoCD, Argo Rollouts +- StatefulSets, Monitoring + +**Exam Alternative (Labs 17-18)** +- Fly.io, 4EVERLAND/IPFS + +
+ +--- + +## How to Submit + +```bash +# 1. Create branch +git checkout -b lab1 + +# 2. Complete lab tasks + +# 3. Commit and push +git add . +git commit -m "Complete lab1" +git push -u origin lab1 + +# 4. Create TWO Pull Requests: +# PR #1: your-fork:lab1 → course-repo:master +# PR #2: your-fork:lab1 → your-fork:master +``` + +
+📝 Submission Checklist + +- [ ] All main tasks completed +- [ ] Documentation files created +- [ ] Screenshots where required +- [ ] Code tested and working +- [ ] Markdown validated ([linter](https://dlaa.me/markdownlint/)) +- [ ] Both PRs created + +
+ +--- + +## Resources + +
+🛠️ Required Tools + +| Tool | Purpose | +|------|---------| +| Git | Version control | +| Docker | Containerization | +| kubectl | Kubernetes CLI | +| Helm | K8s package manager | +| Minikube | Local K8s cluster | +| Terraform | Infrastructure as Code | +| Ansible | Configuration management | + +
+ +
+📚 Documentation Links + +**Core:** +- [Docker](https://docs.docker.com/) +- [Kubernetes](https://kubernetes.io/docs/) +- [Helm](https://helm.sh/docs/) + +**CI/CD:** +- [GitHub Actions](https://docs.github.com/en/actions) +- [Terraform](https://www.terraform.io/docs) +- [Ansible](https://docs.ansible.com/) + +**Observability:** +- [Prometheus](https://prometheus.io/docs/) +- [Grafana](https://grafana.com/docs/) + +**Advanced:** +- [ArgoCD](https://argo-cd.readthedocs.io/) +- [Argo Rollouts](https://argoproj.github.io/argo-rollouts/) +- [HashiCorp Vault](https://developer.hashicorp.com/vault/docs) + +
+ +
+💡 Tips for Success + +1. **Start early** - Don't wait until deadline +2. **Read instructions fully** before starting +3. **Test everything** before submitting +4. **Document as you go** - Don't leave it for the end +5. **Ask questions early** - Don't wait until last minute +6. **Use proper Git workflow** - Branches, commits, PRs + +
+ +
+🔧 Common Issues + +**Docker:** +- Daemon not running → Start Docker Desktop +- Permission denied → Add user to docker group + +**Minikube:** +- Won't start → Try `--driver=docker` +- Resource issues → Allocate more memory/CPU + +**Kubernetes:** +- ImagePullBackOff → Check image name/registry +- CrashLoopBackOff → Check logs: `kubectl logs ` + +
+ +--- + +## Course Completion + +After completing all 16 core labs (+ optional Labs 17-18), you'll have: + +✅ Full-stack DevOps expertise +✅ Production-ready portfolio with 16-18 projects +✅ Container and Kubernetes mastery +✅ CI/CD pipeline experience +✅ Infrastructure as Code skills +✅ Monitoring and observability knowledge +✅ GitOps workflow experience + +--- + +**Ready to begin? Start with [Lab 1](labs/lab01.md)!** + +Questions? Check the course Moodle page or ask during office hours. diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000000..3506a506b7 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,10 @@ +[defaults] +inventory = inventory/hosts.ini +host_key_checking = False +remote_user = ubuntu +retry_files_enabled = False + +[privilege_escalation] +become = True +become_method = sudo +become_user = root \ No newline at end of file diff --git a/ansible/docs/lab5/LAB05.md b/ansible/docs/lab5/LAB05.md new file mode 100644 index 0000000000..bbab8251f7 --- /dev/null +++ b/ansible/docs/lab5/LAB05.md @@ -0,0 +1,104 @@ +# Lab 5 - Ansible Implementation Documentation + +## 1. Architecture Overview +- Ansible version: 2.15+ (tested on macOS with Python 3.10) + +- Target VM OS: Ubuntu 22.04 LTS (5.15.0-151-generic) + +- Role structure: +``` +roles/ +├── common +│ ├── tasks/ +│ │ └── main.yml +│ └── defaults/ +│ └── main.yml +├── docker +│ ├── tasks/ +│ │ └── main.yml +│ ├── handlers/ +│ │ └── main.yml +│ └── defaults/ +│ └── main.yml +└── app_deploy + ├── tasks/ + │ └── main.yml + ├── handlers/ + │ └── main.yml + └── defaults/ + └── main.yml +``` + +- Why roles: Roles separate concerns (system provisioning, Docker setup, app deployment), making playbooks modular, reusable, and easier to maintain. + +## 2. Roles Documentation +**common** +- **Purpose**: Update system, install essential packages, optionally set timezone. +- **Variables**: + - `common_packages` — list of packages (`python3-pip, curl, git, vim, htop`), + - `timezone` — default: `UTC` +- **Handlers**: None +- **Dependencies**: None + +**docker** +- **Purpose**: Install Docker, configure repository, ensure Docker service running, manage user access. +- **Variables**: + - `docker_version` — optional version constraints, + - `docker_user` — user to add to docker group +- **Handlers**: `restart docker` — triggered when Docker config changes +- **Dependencies**: `common` role should run first + +**app_deploy** +- **Purpose**: Pull and run containerized Python app securely using Vault credentials. +- **Variables**: + - `dockerhub_username`, `dockerhub_password` (vaulted); + - `app_name`, `docker_image_tag`, `app_port`, `app_container_name` +- **Handlers**: + - `restart application container` — restarts container if configuration changes +- **Dependencies**: Depends on `docker` role + +## 3. Idempotency Demonstration +**First run of** `provision.yml`: + +![first run](screenshots/first%20run.png) + +**Second run**: + +![first run](screenshots/second%20run.png) + +**Analysis**: +- First run: packages installed, Docker configured, user added — all tasks changed state. +- Second run: nothing changed because tasks checked current state before applying changes. +- **Idempotency**: Roles and tasks are designed to only make changes if the target state differs from the actual state. + +## 4. Ansible Vault Usage +- **Secure storage**: Docker Hub credentials stored in `group_vars/all.yml` encrypted via `ansible-vault`. +- **Vault password strategy**: Use `--ask-vault-pass` or password file (not committed to repo). +- **Importance**: Protects sensitive credentials from accidental exposure in version control. + +## 5. Deployment Verification + +**Playbook run**: +![playbook run](screenshots/deployment%20playbook.png) + +**Container status**: +![container status](screenshots/docker%20ps.png) + +**Health check**: +![main endpoint](screenshots/main%20endpoint.png) +![health check](screenshots/health%20check.png) + +**Handler execution**: The restart handler triggered only when container needed restart. + +## 6. Key Decisions + +- **Why roles**: Keep playbooks modular, easier to maintain, and reusable. +- **Reusability**: Roles can be applied to multiple hosts or projects without duplicating logic. +- **Idempotent tasks**: Ensure they check state before applying changes (apt, docker, etc.). +- **Handlers efficiency**: Avoid unnecessary restarts; only run when a change occurs. +- **Vault necessity**: Keeps sensitive credentials secure, prevents leaks in source control. + +## 7. Challenges +- Docker Hub login failing when vault not loaded → resolved by including `vars_files` in playbook. +- Container inaccessible from outside → fixed by restarting. +- Handlers misconfigured (`state: restarted` not valid) → corrected to `started`. \ No newline at end of file diff --git a/ansible/docs/lab5/screenshots/deployment playbook.png b/ansible/docs/lab5/screenshots/deployment playbook.png new file mode 100644 index 0000000000..fa9ab943ec Binary files /dev/null and b/ansible/docs/lab5/screenshots/deployment playbook.png differ diff --git a/ansible/docs/lab5/screenshots/docker ps.png b/ansible/docs/lab5/screenshots/docker ps.png new file mode 100644 index 0000000000..e1bbc5a0a0 Binary files /dev/null and b/ansible/docs/lab5/screenshots/docker ps.png differ diff --git a/ansible/docs/lab5/screenshots/first run.png b/ansible/docs/lab5/screenshots/first run.png new file mode 100644 index 0000000000..275985726c Binary files /dev/null and b/ansible/docs/lab5/screenshots/first run.png differ diff --git a/ansible/docs/lab5/screenshots/health check.png b/ansible/docs/lab5/screenshots/health check.png new file mode 100644 index 0000000000..6de2eb6a60 Binary files /dev/null and b/ansible/docs/lab5/screenshots/health check.png differ diff --git a/ansible/docs/lab5/screenshots/main endpoint.png b/ansible/docs/lab5/screenshots/main endpoint.png new file mode 100644 index 0000000000..36a61d918a Binary files /dev/null and b/ansible/docs/lab5/screenshots/main endpoint.png differ diff --git a/ansible/docs/lab5/screenshots/second run.png b/ansible/docs/lab5/screenshots/second run.png new file mode 100644 index 0000000000..00948ff538 Binary files /dev/null and b/ansible/docs/lab5/screenshots/second run.png differ diff --git a/ansible/docs/lab6/LAB06.md b/ansible/docs/lab6/LAB06.md new file mode 100644 index 0000000000..f275fb929f --- /dev/null +++ b/ansible/docs/lab6/LAB06.md @@ -0,0 +1,394 @@ +# Lab 6: Advanced Ansible & CI/CD - Submission + +## Overview + +In this lab, I: + +- Refactored Ansible roles using blocks, rescue/always, and tags for flexible task management and error handling. +- Migrated deployment to Docker Compose with a Jinja2 template. +- Implemented safe wipe logic with double gating (variable + tag). +- Set up CI/CD with GitHub Actions for automated deployment. +- Tested all scenarios and included real outputs as evidence. + +**Tech stack:** Ansible 2.16+, Docker Compose v2, GitHub Actions, Jinja2, Ansible Vault. + +--- + +## Task 1: Blocks & Tags (2 pts) + +### Implementation + +- In the `common` role, package installation tasks are grouped in a block with the `packages` tag; user management tasks are in a block with the `users` tag (commented example). +- In the `docker` role, Docker installation tasks are in a block with the `docker_install` tag, configuration tasks in a block with the `docker_config` tag. +- Rescue blocks are used for error handling, always blocks for finalization. +- Tags allow selective execution of tasks. + +#### Example block (roles/common/tasks/main.yml): + +```yaml +- name: Install common packages with error handling + block: + - name: Update apt cache + apt: + update_cache: yes + cache_valid_time: 3600 + - name: Install common packages + apt: + name: "{{ common_packages }}" + state: present + rescue: + - name: "Rescue: Fix apt cache issues" + ansible.builtin.apt: + update_cache: yes + become: true + tags: + - packages + always: + - name: Log package installation completion + ansible.builtin.file: + path: /tmp/common_packages_done + state: touch + tags: + - packages + become: true + tags: + - packages + - common +``` + +#### Example block (roles/docker/tasks/main.yml): + +```yaml +- name: Docker installation block + become: true + tags: + - docker_install + - docker + block: + - name: Add Docker GPG key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + - name: Add Docker repository + apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + state: present + - name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + state: present + update_cache: yes + rescue: + - name: "Rescue: Retry apt update after GPG key failure" + command: apt-get update --fix-missing + become: true + always: + - name: "Always: Ensure Docker service is enabled and started" + service: + name: docker + state: started + enabled: yes +``` + +#### Tag usage examples: + +```bash +ansible-playbook playbooks/provision.yml --tags "docker" +``` +![tags docker](screenshots/--tags%20docker.png) + +```bash +ansible-playbook playbooks/provision.yml --skip-tags "common" +``` +![skip-rags common](screenshots/--skip-tags%20common.png) + +```bash +ansible-playbook playbooks/provision.yml --tags "packages" +``` +![tags packages](screenshots/--tags%20packages.png) + +```bash +ansible-playbook playbooks/provision.yml --tags "docker" --check +``` +![tags docker check](screenshots/--tags%20docker%20check.png) + +```bash +ansible-playbook playbooks/provision.yml --tags "docker_install" +``` +![tags docker install](screenshots/--tags%20docker_install.png) + +list tags: +![list tags](screenshots/list%20tags.png) + +#### Research answers: + +- If the rescue block also fails, the task is marked failed, but the always block still runs. +- Nested blocks are possible but rarely needed. +- Tags on a block are inherited by all tasks inside the block. + +--- + +## Task 2: Docker Compose (3 pts) + +### Implementation + +- Created a Jinja2 template for docker-compose (roles/web_app/templates/docker-compose.yml.j2): + +```yaml +# Docker Compose template for web_app role +# Variables: app_name, docker_image, docker_tag, app_port, app_internal_port, env_vars +version: '{{ docker_compose_version | default("3.8") }}' + +services: + {{ app_name }}: + image: {{ docker_image }}:{{ docker_tag | default('latest') }} + container_name: {{ app_name }} + ports: + - "{{ app_port }}:{{ app_internal_port }}" + environment: + {% for key, value in env_vars.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + restart: unless-stopped + networks: + - webnet + +networks: + webnet: + driver: bridge +``` + +- Added role dependency in meta/main.yml: + +```yaml +--- +dependencies: + - role: docker +``` + +- In tasks/main.yml, deployment is done via docker_compose_v2: + +```yaml +- name: Deploy application with Docker Compose + block: + - name: Create application directory + file: + path: "{{ compose_project_dir }}" + state: directory + mode: "0755" + - name: Copy docker-compose file + template: + src: docker-compose.yml.j2 + dest: "{{ compose_project_dir }}/docker-compose.yml" + - name: Start application + community.docker.docker_compose_v2: + project_src: "{{ compose_project_dir }}" + state: present + when: not (web_app_wipe | bool and 'web_app_wipe' in ansible_run_tags) +``` + +#### Before/After: + +- Before: separate tasks for each container. +- After: single template and docker_compose_v2 module. + +#### Evidence (idempotency, run twice): + +``` +$ ansible-playbook playbooks/deploy.yml --ask-vault-pass +... +TASK [web_app : Stop and remove containers] *********************************************************************************************************** +skipping: [lab04-vm] +... +PLAY RECAP ******************************************************************************************************************************************** +lab04-vm : ok=7 changed=0 unreachable=0 failed=0 skipped=4 rescued=0 ignored=0 + +$ ansible-playbook playbooks/deploy.yml --ask-vault-pass +... +PLAY RECAP ******************************************************************************************************************************************** +lab04-vm : ok=7 changed=0 unreachable=0 failed=0 skipped=4 rescued=0 ignored=0 +``` + +- Second run shows no changes (idempotency). + +![docker ps](screenshots/docker%20ps.png) + +--- + +## Task 3: Wipe Logic (1 pt) + +### Implementation + +- All wipe logic is in roles/web_app/tasks/wipe.yml, included at the top of main.yml via include_tasks. +- Double gating: variable `web_app_wipe` and tag `web_app_wipe`. +- In defaults/main.yml: + +```yaml +web_app_wipe: false # Default: do not wipe +# Set to true to remove application completely +# Wipe only: ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Clean install: ansible-playbook deploy.yml -e "web_app_wipe=true" +``` + +- Example wipe.yml: + +```yaml +- name: Wipe web application + block: + - name: Check if project directory exists + stat: + path: "{{ compose_project_dir }}" + register: project_dir + - name: Stop and remove container directly + community.docker.docker_container: + name: "{{ app_container_name }}" + state: absent + ignore_errors: true + - name: Remove docker-compose.yml file + file: + path: "{{ compose_project_dir }}/docker-compose.yml" + state: absent + - name: Remove application directory + file: + path: "{{ compose_project_dir }}" + state: absent + # - name: Remove Docker images + # community.docker.docker_image: + # name: "{{ docker_image }}:{{ docker_tag | default('latest') }}" + # state: absent + - name: Log wipe completion + debug: + msg: "Application {{ app_name }} wiped successfully" + when: web_app_wipe | bool + tags: + - web_app_wipe +``` + +### Test Results + +#### Clean reinstallation (wipe → deploy): + +``` +$ ansible-playbook playbooks/deploy.yml -e "web_app_wipe=true" --ask-vault-pass +... +TASK [web_app : Include wipe tasks] ******************************************************************************************************************* +included: .../roles/web_app/tasks/wipe.yml for lab04-vm +... +TASK [web_app : Log wipe completion] ****************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-info-python wiped successfully" +} +... +TASK [web_app : Create application directory] ********************************************************************************************************* +changed: [lab04-vm] +... +TASK [web_app : Start application] ******************************************************************************************************************** +changed: [lab04-vm] +... +$ ssh ubuntu@93.77.189.71 "docker ps" +CONTAINER ID IMAGE ... NAMES +d427c81c6df6 gpshfrd/devops-info-python:latest ... devops-info-python +``` + +#### Safety checks: + +``` +$ ansible-playbook playbooks/deploy.yml --tags web_app_wipe --ask-vault-pass +... +TASK [web_app : Include wipe tasks] ******************************************************************************************************************* +included: .../roles/web_app/tasks/wipe.yml for lab04-vm +... +TASK [web_app : Check if project directory exists] **************************************************************************************************** +skipping: [lab04-vm] +... +TASK [web_app : Log wipe completion] ****************************************************************************************************************** +skipping: [lab04-vm] +``` + +- Wipe is skipped if only the tag is set but the variable is not true. + +#### Research answers: + +1. Double gating (variable + tag) prevents accidental wipes. +2. The `never` tag disables tasks entirely; this approach allows controlled execution. +3. Wipe logic must come first to ensure clean reinstall (old removed before new deployed). +4. Clean reinstall is for major upgrades or corrupted state; rolling update is for minor changes. +5. To wipe images/volumes, add tasks using `community.docker.docker_image` and `community.docker.docker_volume` modules. + +--- + +## Task 4: CI/CD (3 pts) + +### Workflow setup + +- Workflow `.github/workflows/ansible-deploy.yml` contains lint, deploy, and verify steps. +- Path filters ensure the workflow only runs on relevant changes. +- Required secrets: ANSIBLE_VAULT_PASSWORD, SSH_PRIVATE_KEY, VM_HOST, VM_USER. +- Example deploy job: + +```yaml +- name: Deploy with Ansible + env: + ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + run: | + cd ansible + echo "$ANSIBLE_VAULT_PASSWORD" > /tmp/vault_pass + ansible-playbook playbooks/deploy.yml \ + -i inventory/hosts.ini \ + --vault-password-file /tmp/vault_pass + rm /tmp/vault_pass +``` + +- Status badge added to README: + +```markdown +[![Ansible Deployment](https://github.com/your-username/your-repo/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/your-username/your-repo/actions/workflows/ansible-deploy.yml) +``` + +#### Evidence: + +- Workflow logs show ansible-lint passing, playbook execution, and verification step. +- Badge is visible in README. +- ![workflow](screenshots/workflow.png) + +#### Research answers: + +1. SSH keys in GitHub Secrets are encrypted and access-controlled, but exposure risk exists if the repo is compromised; rotate keys regularly. +2. Staging → production pipeline: use separate environments, branches, and workflows; promote artifacts after tests pass. +3. For rollbacks: keep previous Compose files/images, add a rollback job to workflow, or use versioned deployments. +4. Self-hosted runners improve security by keeping credentials and code within your infrastructure, reducing exposure to third-party systems. + +--- + +## Task 5: Documentation + +- All modified Ansible files are commented. +- Variables are documented in templates. +- Wipe logic safety is explained in wipe.yml. +- Each CI/CD workflow step is documented in the workflow file. + +--- + +## Testing Results + +- All scenarios (normal deploy, wipe-only, clean reinstall, safety checks) were tested. +- Idempotency: running the playbook twice results in no changes (see run twice evidence above). +- The application is accessible via curl and docker ps (see clean reinstallation evidence above). + +--- + +## Challenges & Solutions + +- Errors with docker_compose when the directory is missing were solved using ignore_errors. +- Double gating was implemented for wipe logic. +- SSH and Vault secrets were configured for CI/CD. +- Idempotency was achieved by using state: present/absent correctly. + +--- + +## Summary + +This lab improved my automation skills with advanced Ansible features and CI/CD. I learned about error handling, selective execution, safe cleanup, and production-grade deployment patterns. The integration with GitHub Actions ensures reliable, repeatable deployments. All scenarios were tested and the application works as expected. Time spent: ~8 hours (don't count breaks). diff --git a/ansible/docs/lab6/screenshots/--skip-tags common.png b/ansible/docs/lab6/screenshots/--skip-tags common.png new file mode 100644 index 0000000000..1369f8128c Binary files /dev/null and b/ansible/docs/lab6/screenshots/--skip-tags common.png differ diff --git a/ansible/docs/lab6/screenshots/--tags docker check.png b/ansible/docs/lab6/screenshots/--tags docker check.png new file mode 100644 index 0000000000..fbd954ef3d Binary files /dev/null and b/ansible/docs/lab6/screenshots/--tags docker check.png differ diff --git a/ansible/docs/lab6/screenshots/--tags docker.png b/ansible/docs/lab6/screenshots/--tags docker.png new file mode 100644 index 0000000000..812bfde52d Binary files /dev/null and b/ansible/docs/lab6/screenshots/--tags docker.png differ diff --git a/ansible/docs/lab6/screenshots/--tags docker_install.png b/ansible/docs/lab6/screenshots/--tags docker_install.png new file mode 100644 index 0000000000..da326304e8 Binary files /dev/null and b/ansible/docs/lab6/screenshots/--tags docker_install.png differ diff --git a/ansible/docs/lab6/screenshots/--tags packages.png b/ansible/docs/lab6/screenshots/--tags packages.png new file mode 100644 index 0000000000..bdc29f3ca2 Binary files /dev/null and b/ansible/docs/lab6/screenshots/--tags packages.png differ diff --git a/ansible/docs/lab6/screenshots/ansible deploy web_app_wipe.png b/ansible/docs/lab6/screenshots/ansible deploy web_app_wipe.png new file mode 100644 index 0000000000..5e8919b24d Binary files /dev/null and b/ansible/docs/lab6/screenshots/ansible deploy web_app_wipe.png differ diff --git a/ansible/docs/lab6/screenshots/ansible deploy.png b/ansible/docs/lab6/screenshots/ansible deploy.png new file mode 100644 index 0000000000..10e598e6f4 Binary files /dev/null and b/ansible/docs/lab6/screenshots/ansible deploy.png differ diff --git a/ansible/docs/lab6/screenshots/docker ps.png b/ansible/docs/lab6/screenshots/docker ps.png new file mode 100644 index 0000000000..4447f5eb8b Binary files /dev/null and b/ansible/docs/lab6/screenshots/docker ps.png differ diff --git a/ansible/docs/lab6/screenshots/list tags.png b/ansible/docs/lab6/screenshots/list tags.png new file mode 100644 index 0000000000..60e61b72ca Binary files /dev/null and b/ansible/docs/lab6/screenshots/list tags.png differ diff --git a/ansible/docs/lab6/screenshots/workflow.png b/ansible/docs/lab6/screenshots/workflow.png new file mode 100644 index 0000000000..1820357846 Binary files /dev/null and b/ansible/docs/lab6/screenshots/workflow.png differ diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000000..7a3e180f7a --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,20 @@ +$ANSIBLE_VAULT;1.1;AES256 +62383930616563623538613238376263636261346431313235336635306364666265396136643735 +3533346236663734363532396636666263373730333636320a393539646163613962323634343835 +37653330636339653435623037336261623138383633383063626139626564616534663737393861 +3361656430633465390a386562333438303162386332343438626335636663613865393837343635 +64616233623831623863633463663262613232623139663761306235666538303038363130376234 +38336534663035633039336239623861383037336134643864633730343931303137633731383735 +38346132666332336239316235353831333363616430626363313133646339306438383037306135 +34663736303464346139663430376436306638313961616335323135653239366230373864346137 +39613438316636323932316132646562323665656433656539636662386163653465623939396138 +32393832323366666465333339303536353935626339653932303732393738396232303265396134 +30333233323765353637303730336536336263343364363966626438396235653237343633646232 +33653161386135663961313161633166396164383064656166613132316136306238666363373434 +62393836323630626131373933326234306234643936353232343763363034356539313066383237 +33613764613132373433663665633134393931646364613632323365396166643238336530636362 +30343438366233646635663135633364636536326333323564313361643933656266653834333466 +65653336386638646639373861323363623938333834343263373363353739366665616363343335 +34373264356665333337623832613761353166346631633966663030393936663163316438666132 +38386639643535623765366366303264343865353531313139333834306638613931636239616334 +666539343762303634333235313535333233 diff --git a/ansible/inventory/hosts.ini b/ansible/inventory/hosts.ini new file mode 100644 index 0000000000..513d43cc24 --- /dev/null +++ b/ansible/inventory/hosts.ini @@ -0,0 +1,2 @@ +[webservers] +lab04-vm ansible_host=158.160.111.187 ansible_user=ubuntu \ No newline at end of file diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000000..b771ad6b9e --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,9 @@ +--- +- name: Deploy application + hosts: webservers + become: true + vars_files: + - ../group_vars/all.yml + + roles: + - web_app diff --git a/ansible/playbooks/provision.yml b/ansible/playbooks/provision.yml new file mode 100644 index 0000000000..6334c412cc --- /dev/null +++ b/ansible/playbooks/provision.yml @@ -0,0 +1,12 @@ +--- +- name: Provision web servers + hosts: webservers + become: true + + roles: + - role: common + tags: + - common + - role: docker + tags: + - docker diff --git a/ansible/playbooks/roles/app_deploy/tasks/main.yml b/ansible/playbooks/roles/app_deploy/tasks/main.yml new file mode 100644 index 0000000000..ada4c646bd --- /dev/null +++ b/ansible/playbooks/roles/app_deploy/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Login to Docker Hub + community.docker.docker_login: + username: "{{ dockerhub_username }}" + password: "{{ dockerhub_password }}" + no_log: true + +- name: Pull Docker image + community.docker.docker_image: + name: "{{ docker_image }}" + tag: "{{ docker_image_tag }}" + source: pull + +- name: Stop existing container if running + community.docker.docker_container: + name: "{{ app_container_name }}" + state: stopped + image: "{{ docker_image }}:{{ docker_image_tag }}" + failed_when: false + +- name: Remove old container if exists + community.docker.docker_container: + name: "{{ app_container_name }}" + state: absent + failed_when: false + +- name: Run new container + community.docker.docker_container: + name: "{{ app_container_name }}" + image: "{{ docker_image }}:{{ docker_image_tag }}" + state: started + restart_policy: "{{ restart_policy }}" + published_ports: + - "{{ app_port }}:{{ app_port }}" + env: "{{ env_vars }}" + notify: restart application container + +- name: Wait for application to be ready + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ app_port }}" + timeout: 60 + +- name: Verify health endpoint + ansible.builtin.uri: + url: "http://localhost:{{ app_port }}/health" + status_code: 200 diff --git a/ansible/playbooks/roles/common/defaults/main.yml b/ansible/playbooks/roles/common/defaults/main.yml new file mode 100644 index 0000000000..2282a109a1 --- /dev/null +++ b/ansible/playbooks/roles/common/defaults/main.yml @@ -0,0 +1,7 @@ +--- +common_packages: + - python3-pip + - curl + - git + - vim + - htop diff --git a/ansible/playbooks/roles/common/tasks/main.yml b/ansible/playbooks/roles/common/tasks/main.yml new file mode 100644 index 0000000000..629d7ab7bd --- /dev/null +++ b/ansible/playbooks/roles/common/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Install common packages with error handling + become: true + tags: + - packages + - common + block: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install common packages + ansible.builtin.apt: + name: "{{ common_packages }}" + state: present + rescue: + - name: "Rescue: Fix apt cache issues" + ansible.builtin.apt: + update_cache: true + become: true + tags: + - packages + always: + - name: Log package installation completion + ansible.builtin.file: + path: /tmp/common_packages_done + state: touch + mode: "0644" + tags: + - packages diff --git a/ansible/playbooks/roles/docker/defaults/main.yml b/ansible/playbooks/roles/docker/defaults/main.yml new file mode 100644 index 0000000000..fcddca27ee --- /dev/null +++ b/ansible/playbooks/roles/docker/defaults/main.yml @@ -0,0 +1,7 @@ +--- +docker_packages: + - docker-ce + - docker-ce-cli + - containerd.io +docker_user: ubuntu +docker_version: "" diff --git a/ansible/playbooks/roles/docker/handlers/main.yml b/ansible/playbooks/roles/docker/handlers/main.yml new file mode 100644 index 0000000000..07aa0eb290 --- /dev/null +++ b/ansible/playbooks/roles/docker/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart docker + ansible.builtin.service: + name: docker + state: restarted diff --git a/ansible/playbooks/roles/docker/tasks/main.yml b/ansible/playbooks/roles/docker/tasks/main.yml new file mode 100644 index 0000000000..f7f0af09b2 --- /dev/null +++ b/ansible/playbooks/roles/docker/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Docker installation block + become: true + tags: + - docker_install + - docker + block: + - name: Add Docker GPG key + ansible.builtin.apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: Add Docker repository + ansible.builtin.apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + state: present + + - name: Install Docker packages + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + state: present + update_cache: true + rescue: + - name: "Rescue: Retry apt update after GPG key failure" + ansible.builtin.apt: + update_cache: true + become: true + always: + - name: "Always: Ensure Docker service is enabled and started" + ansible.builtin.service: + name: docker + state: started + enabled: true + +- name: Docker configuration block + become: true + tags: + - docker_config + - docker + block: + - name: Add user to docker group + ansible.builtin.user: + name: "{{ docker_user }}" + groups: docker + append: true + + - name: Install python3-docker + ansible.builtin.pip: + name: docker + state: present + notify: restart docker diff --git a/ansible/playbooks/roles/web_app/defaults/main.yml b/ansible/playbooks/roles/web_app/defaults/main.yml new file mode 100644 index 0000000000..3c9aa6c5b8 --- /dev/null +++ b/ansible/playbooks/roles/web_app/defaults/main.yml @@ -0,0 +1,15 @@ +--- +web_app_app_port: 5000 +web_app_app_internal_port: "{{ web_app_app_port }}" +web_app_app_external_port: "{{ web_app_app_port }}" +web_app_app_container_name: "{{ web_app_app_name | default('devops-info-python') }}" +web_app_compose_project_dir: "/opt/devops-info-python" +web_app_docker_image: gpshfrd/devops-info-python +web_app_docker_image_tag: latest +web_app_restart_policy: unless-stopped +web_app_env_vars: {} + +web_app_wipe: false # Default: do not wipe +# Set to true to remove application completely +# Wipe only: ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Clean install: ansible-playbook deploy.yml -e "web_app_wipe=true" diff --git a/ansible/playbooks/roles/web_app/handlers/main.yml b/ansible/playbooks/roles/web_app/handlers/main.yml new file mode 100644 index 0000000000..33751e6230 --- /dev/null +++ b/ansible/playbooks/roles/web_app/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Restart application container + community.docker.docker_container: + name: "{{ web_app_app_container_name }}" + image: "{{ web_app_docker_image }}:{{ web_app_docker_image_tag }}" + state: started + restart_policy: "{{ web_app_restart_policy }}" + published_ports: + - "{{ web_app_app_port }}:{{ web_app_app_port }}" + env: "{{ web_app_env_vars }}" + recreate: true diff --git a/ansible/playbooks/roles/web_app/meta/main.yml b/ansible/playbooks/roles/web_app/meta/main.yml new file mode 100644 index 0000000000..a1e4ac0a65 --- /dev/null +++ b/ansible/playbooks/roles/web_app/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: docker + +collections: + - community.docker diff --git a/ansible/playbooks/roles/web_app/tasks/main.yml b/ansible/playbooks/roles/web_app/tasks/main.yml new file mode 100644 index 0000000000..415328cfc2 --- /dev/null +++ b/ansible/playbooks/roles/web_app/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Include wipe tasks + ansible.builtin.include_tasks: wipe.yml + tags: + - web_app_wipe + +- name: Deploy application with Docker Compose + when: not (web_app_wipe | bool and 'web_app_wipe' in ansible_run_tags) + block: + - name: Create application directory + ansible.builtin.file: + path: "{{ web_app_compose_project_dir }}" + state: directory + mode: "0755" + + - name: Copy docker-compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ web_app_compose_project_dir }}/docker-compose.yml" + mode: "0755" + + - name: Start application + community.docker.docker_compose_v2: + project_src: "{{ web_app_compose_project_dir }}" + state: present diff --git a/ansible/playbooks/roles/web_app/tasks/wipe.yml b/ansible/playbooks/roles/web_app/tasks/wipe.yml new file mode 100644 index 0000000000..4a4c8b36b6 --- /dev/null +++ b/ansible/playbooks/roles/web_app/tasks/wipe.yml @@ -0,0 +1,35 @@ +--- +- name: Wipe web application + when: web_app_wipe | bool + tags: + - web_app_wipe + block: + - name: Check if project directory exists + ansible.builtin.stat: + path: "{{ web_app_compose_project_dir }}" + register: web_app_project_dir + + - name: Stop and remove container directly + community.docker.docker_container: + name: "{{ web_app_app_container_name }}" + state: absent + failed_when: false + + - name: Remove docker-compose.yml file + ansible.builtin.file: + path: "{{ web_app_compose_project_dir }}/docker-compose.yml" + state: absent + + - name: Remove application directory + ansible.builtin.file: + path: "{{ web_app_compose_project_dir }}" + state: absent + + # - name: Remove Docker images + # community.docker.docker_image: + # name: "{{ web_app_docker_image }}:{{ web_app_docker_image_tag | default('latest') }}" + # state: absent + + - name: Log wipe completion + ansible.builtin.debug: + msg: "Application {{ web_app_app_name | default('devops-info-python') }} wiped successfully" diff --git a/ansible/playbooks/roles/web_app/templates/docker-compose.yml.j2 b/ansible/playbooks/roles/web_app/templates/docker-compose.yml.j2 new file mode 100644 index 0000000000..9dac3bf068 --- /dev/null +++ b/ansible/playbooks/roles/web_app/templates/docker-compose.yml.j2 @@ -0,0 +1,21 @@ +# Docker Compose template for web_app role +# Variables: web_app_app_name, web_app_docker_image, web_app_docker_image_tag, web_app_app_port, web_app_app_internal_port, web_app_env_vars +version: '{{ docker_compose_version | default("3.8") }}' + +services: + {{ web_app_app_name | default('devops-info-python') }}: + image: {{ web_app_docker_image }}:{{ web_app_docker_image_tag | default('latest') }} + container_name: {{ web_app_app_name | default('devops-info-python') }} + ports: + - "{{ web_app_app_port }}:{{ web_app_app_internal_port | default(web_app_app_port) }}" + environment: + {% for key, value in web_app_env_vars.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + restart: unless-stopped + networks: + - webnet + +networks: + webnet: + driver: bridge diff --git a/app_go/.gitignore b/app_go/.gitignore new file mode 100644 index 0000000000..9102a5aa0c --- /dev/null +++ b/app_go/.gitignore @@ -0,0 +1 @@ +devops-info diff --git a/app_go/README.md b/app_go/README.md new file mode 100644 index 0000000000..ef223cdf6b --- /dev/null +++ b/app_go/README.md @@ -0,0 +1,22 @@ +# DevOps Info Service (Go) + +## Overview +Compiled implementation of the DevOps Info Service using Go. +Designed for efficient containerization and minimal runtime footprint. + +## Requirements +- Go 1.22+ + +## Build +```bash +go build -o devops-info +``` + +## Run +```bash +./devops-info +``` + +## Endpoints +- `GET /` +- `GET /health` \ No newline at end of file diff --git a/app_go/docs/GO.md b/app_go/docs/GO.md new file mode 100644 index 0000000000..b2a73f8eb3 --- /dev/null +++ b/app_go/docs/GO.md @@ -0,0 +1,5 @@ +## Why Go + +Go was chosen for the compiled implementation due to its simplicity, fast compilation, and suitability for containerized environments. +It produces a single static binary, which makes it ideal for multi-stage Docker builds and minimal runtime images. +Go is widely used in DevOps tooling, including Docker, Kubernetes, and Prometheus. \ No newline at end of file diff --git a/app_go/docs/LAB01.md b/app_go/docs/LAB01.md new file mode 100644 index 0000000000..9086d88b6b --- /dev/null +++ b/app_go/docs/LAB01.md @@ -0,0 +1,17 @@ +## Implementation Details + +The Go version implements the same API structure as the Python service using the standard `net/http` package. + +## Build Process +```bash +go build -o devops-info +``` + +## Binary Size Comparison + +| Implementation | Size | +| -------------- | ---------------------------- | +| Python (Flask) | ~30–50 MB (with venv & deps) | +| Go binary | ~7–10 MB | + +The Go binary does not require an interpreter or external dependencies, making it more efficient for deployment. diff --git a/app_go/docs/screenshots/01-main-endpoint.png b/app_go/docs/screenshots/01-main-endpoint.png new file mode 100644 index 0000000000..8a1fd0664a Binary files /dev/null and b/app_go/docs/screenshots/01-main-endpoint.png differ diff --git a/app_go/docs/screenshots/02-health-endpoint.png b/app_go/docs/screenshots/02-health-endpoint.png new file mode 100644 index 0000000000..3198a1d51b Binary files /dev/null and b/app_go/docs/screenshots/02-health-endpoint.png differ diff --git a/app_go/go.mod b/app_go/go.mod new file mode 100644 index 0000000000..7a7fcedd1c --- /dev/null +++ b/app_go/go.mod @@ -0,0 +1,3 @@ +module devops-info-service + +go 1.22 diff --git a/app_go/main.go b/app_go/main.go new file mode 100644 index 0000000000..ad518ebfd9 --- /dev/null +++ b/app_go/main.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "runtime" + "time" +) + +var startTime = time.Now().UTC() + +type Service struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Framework string `json:"framework"` +} + +type System struct { + Hostname string `json:"hostname"` + Platform string `json:"platform"` + Architecture string `json:"architecture"` + CPUCount int `json:"cpu_count"` + GoVersion string `json:"go_version"` +} + +type RuntimeInfo struct { + UptimeSeconds int `json:"uptime_seconds"` + UptimeHuman string `json:"uptime_human"` + CurrentTime string `json:"current_time"` + Timezone string `json:"timezone"` +} + +type RequestInfo struct { + Method string `json:"method"` + Path string `json:"path"` +} + +type Endpoint struct { + Path string `json:"path"` + Method string `json:"method"` + Description string `json:"description"` +} + +type ServiceInfo struct { + Service Service `json:"service"` + System System `json:"system"` + Runtime RuntimeInfo `json:"runtime"` + Request RequestInfo `json:"request"` + Endpoints []Endpoint `json:"endpoints"` +} + +func getUptime() (int, string) { + seconds := int(time.Since(startTime).Seconds()) + hours := seconds / 3600 + minutes := (seconds % 3600) / 60 + return seconds, fmt.Sprintf("%d hours %d minutes", hours, minutes) +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + hostname, _ := os.Hostname() + uptimeSeconds, uptimeHuman := getUptime() + + info := ServiceInfo{ + Service: Service{ + Name: "devops-info-service", + Version: "1.0.0", + Description: "DevOps Course Info Service", + Framework: "Go net/http", + }, + System: System{ + Hostname: hostname, + Platform: runtime.GOOS, + Architecture: runtime.GOARCH, + CPUCount: runtime.NumCPU(), + GoVersion: runtime.Version(), + }, + Runtime: RuntimeInfo{ + UptimeSeconds: uptimeSeconds, + UptimeHuman: uptimeHuman, + CurrentTime: time.Now().UTC().Format(time.RFC3339), + Timezone: "UTC", + }, + Request: RequestInfo{ + Method: r.Method, + Path: r.URL.Path, + }, + Endpoints: []Endpoint{ + {Path: "/", Method: "GET", Description: "Main service information endpoint"}, + {Path: "/health", Method: "GET", Description: "Health check endpoint"}, + }, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(info) +} + +func healthHandler(w http.ResponseWriter, r *http.Request) { + uptimeSeconds, _ := getUptime() + + response := map[string]interface{}{ + "status": "healthy", + "timestamp": time.Now().UTC().Format(time.RFC3339), + "uptime_seconds": uptimeSeconds, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func main() { + http.HandleFunc("/", mainHandler) + http.HandleFunc("/health", healthHandler) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + + http.ListenAndServe(":"+port, nil) +} diff --git a/app_python/.dockerignore b/app_python/.dockerignore new file mode 100644 index 0000000000..062d1e4670 --- /dev/null +++ b/app_python/.dockerignore @@ -0,0 +1,15 @@ +__pycache__/ +*.pyc +*.pyo +*.pyd + +.git/ +.gitignore + +venv/ +.venv/ + +docs/ +tests/ + +README.md \ No newline at end of file diff --git a/app_python/.gitignore b/app_python/.gitignore new file mode 100644 index 0000000000..4de420a8f7 --- /dev/null +++ b/app_python/.gitignore @@ -0,0 +1,12 @@ +# Python +__pycache__/ +*.py[cod] +venv/ +*.log + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store \ No newline at end of file diff --git a/app_python/Dockerfile b/app_python/Dockerfile new file mode 100644 index 0000000000..602ef154d9 --- /dev/null +++ b/app_python/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Non-root user setup +RUN useradd --create-home appuser + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . + +#Swithh to non-root user +USER appuser + +EXPOSE 8000 + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/app_python/README.md b/app_python/README.md new file mode 100644 index 0000000000..8d17200a4b --- /dev/null +++ b/app_python/README.md @@ -0,0 +1,54 @@ +# DevOps Info Service + +[![Python CI](https://github.com/Gpshfrd/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg)](https://github.com/Gpshfrd/DevOps-Core-Course/actions/workflows/python-ci.yml) + +## Overview +A lightweight web service, that exposes the system, runtime, and request information. +Used as a foundation for DevOps labs (Docker, CI/CD, monitoring). + +## Prerequisites +- Python 3.11+ +- pip + +## Installation +```bash +python -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +## Running the Application +```bash +python app.py +# Or with custom config +PORT=8080 python app.py +``` + +## API Endpoints +- `GET /` - Service and system information +- `GET /health` - Health check + +## Configuration + +| Variable | Default | Description | +| ---------- | --------- | ------------------ | +| `HOST` | `0.0.0.0` | Bind address | +| `PORT` | `5000` | Server port number | +| `DEBUG` | `false` | Debug mode | + +## Docker + +Build image locally: +```bash +docker build -t devops-info-python . +``` + +Run container: +```bash +docker run -p 8000:8000 devops-info-python +``` + +Pull from Docker Hub: +```bash +docker pull gpshfrd/devops-info-python:1.0.0 +``` \ No newline at end of file diff --git a/app_python/app.py b/app_python/app.py new file mode 100644 index 0000000000..a684564520 --- /dev/null +++ b/app_python/app.py @@ -0,0 +1,230 @@ +import os +import platform +import socket +import logging +import sys +import time +from datetime import datetime, timezone +from pythonjsonlogger import jsonlogger + +from flask import Flask, jsonify, request, Response +from prometheus_client import Counter, Histogram, Gauge, generate_latest + +### Configuration + +HOST = os.getenv("HOST", "0.0.0.0") +PORT = int(os.getenv("PORT", 5000)) +DEBUG = os.getenv("DEBUG", "False").lower() == "true" + +SERVICE_NAME = "devops-info-service" +SERVICE_VERSION = "1.0.0" +SERVICE_DESCRIPTION = "DevOps course info service" +FRAMEWORK = "Flask" + +### App and logging +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +logHandler = logging.StreamHandler(sys.stdout) + +formatter = jsonlogger.JsonFormatter( + "%(asctime)s %(levelname)s %(message)s %(method)s %(path)s %(status)s %(client_ip)s" +) + +logHandler.setFormatter(formatter) +logger.addHandler(logHandler) + +app = Flask(__name__) + +START_TIME = datetime.now(timezone.utc) + +### Prometeus metrics + +http_requests_total = Counter( + "http_requests_total", + "Total HTTP requests", + ["method", "endpoint", "status"] +) + +http_request_duration_seconds = Histogram( + "http_request_duration_seconds", + "HTTP request duration in seconds", + ["method", "endpoint"] +) + +http_requests_in_progress = Gauge( + "http_requests_in_progress", + "Number of HTTP requests in progress" +) + +### Helper functions + +def get_uptime(): + delta = datetime.now(timezone.utc) - START_TIME + seconds = int(delta.total_seconds()) + minutes = (seconds % 3600) // 60 + hours = seconds // 3600 + return seconds, f"{hours} hours, {minutes} minutes" + +def get_system_info(): + return { + "hostname": socket.gethostname(), + "platform": platform.system(), + "platform-version": platform.version(), + "architecture": platform.machine(), + "cpu-count": os.cpu_count(), + "python-version": platform.python_version(), + } + +### Routes + +@app.route("/", methods=["GET"]) +def index(): + uptime_seconds, uptime_human = get_uptime() + + logger.info("Main endpoint accessed") + + response = { + "service": { + "name": SERVICE_NAME, + "version": SERVICE_VERSION, + "description": SERVICE_DESCRIPTION, + "framework": FRAMEWORK + }, + "system": get_system_info(), + "runtime": { + "uptime_seconds": uptime_seconds, + "uptime_human": uptime_human, + "current_time": datetime.now(timezone.utc).isoformat(), + "timezone": "UTC" + }, + "request": { + "client_ip": request.remote_addr, + "user_agent": request.headers.get("User-Agent"), + "method": request.method, + "path": request.path + }, + "endpoints": [ + {"path": "/", "method": "GET", "description": "Service information"}, + {"path": "/health", "method": "GET", "description": "Health check"} + ], + } + + return jsonify(response) + +@app.route("/health", methods=["GET"]) +def health(): + uptime_seconds, _ = get_uptime() + + return jsonify({ + "status": "healthy", + "timestamp": datetime.now(timezone.utc).isoformat(), + "uptime_seconds": uptime_seconds, + }) + +@app.route("/metrics", methods=["GET"]) +def metrics(): + return generate_latest(), 200, {"Content-Type": "text/plain"} + +@app.route("/ready", methods=["GET"]) +def readiness(): + is_ready = True + ready_details = { + "status": "ready" if is_ready else "not ready", + "timestamp": datetime.now(timezone.utc).isoformat(), + "checks": { + "app": "running", + "dependencies": { + "database": "not_configured", + "cache": "not_configured" + } + } + } + + if not is_ready: + return jsonify(ready_details), 503 + + return jsonify(ready_details), 200 + + +### Error Handlers +@app.errorhandler(404) +def not_found(error): + return jsonify({"error": "Not Found", "message": "Endpoint does not exist."}), 404 + +@app.errorhandler(500) +def internal_error(error): + return jsonify({"error": "Internal Server Error", "message": "An unexpected error occurred."}), 500 + +@app.errorhandler(Exception) +def handle_error(e): + logger.error( + "Unhandled exception", + extra={ + "method": request.method if request else "-", + "path": request.path if request else "-", + "status": 500, + "client_ip": request.remote_addr if request else "-", + }, + exc_info=True + ) + return {"error": "internal server error"}, 500 + +### Logging middleware + +@app.before_request +def log_request(): + request.start_time = time.time() + + http_requests_in_progress.inc() + + logger.info( + "Request received", + extra={ + "method": request.method, + "path": request.path, + "client_ip": request.remote_addr, + }, + ) + +@app.after_request +def log_response(response): + duration = time.time() - request.start_time + + http_requests_total.labels( + method=request.method, + endpoint=request.path, + status=response.status_code + ).inc() + + http_request_duration_seconds.labels( + method=request.method, + endpoint=request.path + ).observe(duration) + + http_requests_in_progress.dec() + + logger.info( + "Response sent", + extra={ + "status": response.status_code, + "path": request.path, + "method": request.method, + "client_ip": request.remote_addr, + }, + ) + return response + +### Entrypoint +if __name__ == "__main__": + logger.info( + "Application started", + extra={ + "method": "-", + "path": "-", + "status": "-", + "client_ip": "-", + "port": PORT + } + ) + app.run(host=HOST, port=PORT, debug=DEBUG) \ No newline at end of file diff --git a/app_python/docs/LAB01.md b/app_python/docs/LAB01.md new file mode 100644 index 0000000000..559a1290be --- /dev/null +++ b/app_python/docs/LAB01.md @@ -0,0 +1,110 @@ +# Lab 1 - DevOps Info Service: Web Application Development + +## Framework selection +For this lab, I chose **Flask** as the web framework. +Flask is a lightweight and minimalistic framework that provides full control over request handling and application structure. + +I had prior experience working with Flask, which allowed me to focus on the DevOps-related goals of the assignment rather than spending time learning a new framework. + +Flask is well-suited for small services and internal tools, which aligns with the purpose of this DevOps Info Service. Its simplicity makes the application easier to understand, maintain, containerize, and monitor in later labs. + +## Best Practices Applied +- PEP 8 +- Centralized config via env vars +- Structured logging +- Explicit error handlers + +## API Documentation +### Request / response examples: +#### 1. Main Endpoint `Get /` + +**Request:** +```bash +curl http://localhost:5000 | jq +``` + +**Response Example:** +```json +{ + "endpoints": [ + { + "description": "Service information", + "method": "GET", + "path": "/" + }, + { + "description": "Health check", + "method": "GET", + "path": "/health" + } + ], + "request": { + "client_ip": "127.0.0.1", + "method": "GET", + "path": "/", + "user_agent": "curl/8.7.1" + }, + "runtime": { + "current_time": "2026-01-27T12:16:29.586466+00:00", + "timezone": "UTC", + "uptime_human": "0 hours, 0 minutes", + "uptime_seconds": 7 + }, + "service": { + "description": "DevOps course info service", + "framework": "Flask", + "name": "devops-info-service", + "version": "1.0.0" + }, + "system": { + "architecture": "x86_64", + "cpu-count": 8, + "hostname": "MacBook-Pro-Aliia.local", + "platform": "Darwin", + "platform-version": "Darwin Kernel Version 24.6.0: Wed Nov 5 21:30:23 PST 2025; root:xnu-11417.140.69.705.2~1/RELEASE_X86_64", + "python-version": "3.11.0" + } +} +``` + +#### 2. Health check `Get /health` + +**Request:** +```bash +curl http://localhost:5000 | jq +``` + +**Response Example:** +```json +{ + "status": "healthy", + "timestamp": "2026-01-27T13:44:02.759040+00:00", + "uptime_seconds": 5260 +} +``` + +## Testing Evidence +### Main Endpoint: +![01-main-endpoint](screenshots/lab01/01-main-endpoint.png) +### Health Check: +![02-health-check](screenshots/lab01/02-health-check.png) +### Formatted Output: +![03-formatted-output](screenshots/lab01/03-formatted-output.png) + +## Challenges & Solutions +### Correct Uptime Calculation +**Problem:** +The application is required to report accurate uptime information in seconds and in a human-readable format. A naive implementation based on system uptime or repeated timestamp calculations can lead to inconsistent results, especially after application restarts or during long-running sessions. + +**Solution:** +To ensure consistent and deterministic uptime values, the application records a fixed START_TIME at launch using UTC time. Uptime is then calculated as the difference between the current UTC timestamp and this initial start time. +This approach guarantees stable and reproducible uptime values and avoids issues related to local time zones or system clock changes. + +## GitHub Community +### Importance of Starring Repositories +Starring repositories on GitHub serves both practical and community purposes. From a personal perspective, stars act as bookmarks that make it easier to return to useful tools and references. From a community perspective, stars provide a visible signal of interest and trust, helping high-quality projects gain visibility and attract contributors. +For open-source maintainers, stars also function as feedback and motivation to continue development. + +### Value of Following Developers and Classmates +Following developers on GitHub allows staying informed about their activity, projects, and coding practices. This helps with learning through real-world examples and understanding how others approach problem-solving. +Following classmates supports collaboration within the course by making it easier to discover their work, exchange ideas, and build connections that can be useful in future team-based projects and professional development. \ No newline at end of file diff --git a/app_python/docs/LAB02.md b/app_python/docs/LAB02.md new file mode 100644 index 0000000000..c5b7766396 --- /dev/null +++ b/app_python/docs/LAB02.md @@ -0,0 +1,56 @@ +# Lab 2 - DevOps Info Service: Docker Containerization + +## Docker Best Practices Applied +**Non-root user** +The container runs under a non-root user created inside the image, which reduces security risks by limiting container privileges in case of compromise. + +**Layer caching** +`requirements.txt` is copied before application code to leverage Docker layer caching, which significantly speeds up rebuilds when application code changes, but dependencies do not. + +## Image information and Decisions +Base image: python:3.13-slim +Reason: smaller image size comparing with full python image while keeping glibc compatibility. + +Final image size: 182.36MB +Acceptable for a python service with external dependencies. + +## Build and Run Processes +### Docker build +![docker-build](screenshots/lab02/docker-build.png) + +### Docker run +![docker-run](screenshots/lab02/docker-run.png) + +### Docker push +![docker-push](screenshots/lab02/docker-push.png) + +### Check +![main-endpoint](screenshots/lab02/main-endpoint.png) +![health-check](screenshots/lab02/health-check.png) + +### Docker Hub URL +https://hub.docker.com/r/gpshfrd/devops-info-python + +## Technical Analysis +### Why does your Dockerfile work the way it does? +The Dockerfile is written step by step so Docker can build the image correctly and efficiently. +A base image is chosen first, then the working directory is set. Dependencies are installed before copying the application code so Docker can reuse cached layers. +The application is started with a clear command, so the container always runs the same way. + +### What would happen if you changed the layer order? +Docker uses caching. +If the source code is copied before installing dependencies, Docker will reinstall dependencies every time the code changes. This makes builds slower. Correct layer order helps Docker reuse layers and build images faster. + +### What security considerations did you implement? +- The application runs as a non-root user +- A minimal base image is used +- Only necessary files are copied into the container + +### How does .dockerignore improve your build? +`.dockerignore` prevents unnecessary files from being included in the Docker build, which makes the build faster, reduces image size, and avoids copying files that are not needed for running the app. + +## Challenges and Solutions +Initially, I tried to launch the container using the wrong name of the local image. I was also a little confused with the port mapping, so curl returned connection errors. + +When starting the container, I rechecked the name of the image with the tag (docker run -p 8000:5000 username/image:tag). +Also I checked port mapping between container and host to make sure endpoints were reachable. \ No newline at end of file diff --git a/app_python/docs/LAB03.md b/app_python/docs/LAB03.md new file mode 100644 index 0000000000..74376504f9 --- /dev/null +++ b/app_python/docs/LAB03.md @@ -0,0 +1,63 @@ +# Lab 3 — Continuous Integration (CI/CD) + +## 1. Overview + +**Testing Framework:** 'pytest' — simple syntax, powerful fixtures, convenient integration with CI. + +**Endpoints Covered:** +- `GET /` — checking the JSON structure and the presence of fields +- `GET /health` — checking the health status of the app + +**CI Workflow Triggers:** +- Push and pull request for `app_python/**` +- Workflow starts only when the Python application code is changed + +**Versioning Strategy:** +- **CalVer (Calendar Versioning)** — the `YYYY.MM.DD` format version +- Automatically generated from the current date +- Docker tags: `latest` and `YYYY.MM.DD` + +--- + +## 2. Workflow Evidence + +**GitHub Actions CI Run:** [Link to successful workflow](https://github.com/Gpshfrd/DevOps-Core-Course/actions/runs/21863939737) +![run-complete](screenshots/lab03/run_complete.png) + +**Local Test Run:** +![tests-passed](screenshots/lab03/tests_passed.png) + +Docker Image on Docker Hub: +![tags](screenshots/lab03/tags.png) + +[![Python CI](https://github.com/Gpshfrd/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg)](https://github.com/Gpshfrd/DevOps-Core-Course/actions/workflows/python-ci.yml) displayed at the top of the README, shows the current CI status (passing/failing) + +## 3. Best Practices Implemented + +- **Dependency Caching** — speeds up the installation of Python dependencies: +`actions/setup-python@v5 with cache: pip` +**Speed improvement**: from 15s to 5s on repeated launches (approximately 3x acceleration) +- **Fail Fast** — workflow stops at the first error, saving CI time +- **Conditional Steps** — Docker image build and push are performed only on the main branch +- **Snyk Security Scan** — vulnerabilities found in the requests package (low/medium), fixed by version update: +```yaml + - name: Run Snyk security scan + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + run: | + pip install snyk + snyk test --severity-threshold=medium +``` +- **Status Badge** — shows the current workflow status directly in the README + +## 4. Key Decisions + +- **Versioning Strategy**: CalVer — convenient for continuous deployment and easy to compare build dates +- **Docker Tags**: two tags are created: latest and YYYY.MM.DD for historicity and identification of builds +- **Workflow Triggers**: push and pull request in Python code — saves resources, CI does not run on changes in other directories +- **Test Coverage**: the main functional endpoints are covered, utilities and auxiliary scripts are excluded; the current coverage level is ~85% + +## 5. Challenges +- Snyk API Token Search: Personal Access Token (PAT) from Snyk account is now used → added as SNYK_TOKEN secret +- Configuring caching: it was necessary to specify the correct path for pip cache +- Docker images Versioning: CalVer is easier to automate than SemVer for daily builds \ No newline at end of file diff --git a/app_python/docs/screenshots/lab01/01-main-endpoint.png b/app_python/docs/screenshots/lab01/01-main-endpoint.png new file mode 100644 index 0000000000..f1885c3a70 Binary files /dev/null and b/app_python/docs/screenshots/lab01/01-main-endpoint.png differ diff --git a/app_python/docs/screenshots/lab01/02-health-check.png b/app_python/docs/screenshots/lab01/02-health-check.png new file mode 100644 index 0000000000..d851293efa Binary files /dev/null and b/app_python/docs/screenshots/lab01/02-health-check.png differ diff --git a/app_python/docs/screenshots/lab01/03-formatted-output.png b/app_python/docs/screenshots/lab01/03-formatted-output.png new file mode 100644 index 0000000000..d45bd4a7db Binary files /dev/null and b/app_python/docs/screenshots/lab01/03-formatted-output.png differ diff --git a/app_python/docs/screenshots/lab02/docker-build.png b/app_python/docs/screenshots/lab02/docker-build.png new file mode 100644 index 0000000000..00bef19aea Binary files /dev/null and b/app_python/docs/screenshots/lab02/docker-build.png differ diff --git a/app_python/docs/screenshots/lab02/docker-push.png b/app_python/docs/screenshots/lab02/docker-push.png new file mode 100644 index 0000000000..007314b2bb Binary files /dev/null and b/app_python/docs/screenshots/lab02/docker-push.png differ diff --git a/app_python/docs/screenshots/lab02/docker-run.png b/app_python/docs/screenshots/lab02/docker-run.png new file mode 100644 index 0000000000..607c91af30 Binary files /dev/null and b/app_python/docs/screenshots/lab02/docker-run.png differ diff --git a/app_python/docs/screenshots/lab02/health-check.png b/app_python/docs/screenshots/lab02/health-check.png new file mode 100644 index 0000000000..5edb11b825 Binary files /dev/null and b/app_python/docs/screenshots/lab02/health-check.png differ diff --git a/app_python/docs/screenshots/lab02/main-endpoint.png b/app_python/docs/screenshots/lab02/main-endpoint.png new file mode 100644 index 0000000000..64201a8e8c Binary files /dev/null and b/app_python/docs/screenshots/lab02/main-endpoint.png differ diff --git a/app_python/docs/screenshots/lab03/run_complete.png b/app_python/docs/screenshots/lab03/run_complete.png new file mode 100644 index 0000000000..6fbea445a2 Binary files /dev/null and b/app_python/docs/screenshots/lab03/run_complete.png differ diff --git a/app_python/docs/screenshots/lab03/tags.png b/app_python/docs/screenshots/lab03/tags.png new file mode 100644 index 0000000000..ac51c6a82e Binary files /dev/null and b/app_python/docs/screenshots/lab03/tags.png differ diff --git a/app_python/docs/screenshots/lab03/tests_passed.png b/app_python/docs/screenshots/lab03/tests_passed.png new file mode 100644 index 0000000000..cae6f15648 Binary files /dev/null and b/app_python/docs/screenshots/lab03/tests_passed.png differ diff --git a/app_python/requirements.txt b/app_python/requirements.txt new file mode 100644 index 0000000000..8ec9830901 --- /dev/null +++ b/app_python/requirements.txt @@ -0,0 +1,4 @@ +Flask==3.1.0 +pytest +python-json-logger +prometheus-client==0.23.1 \ No newline at end of file diff --git a/app_python/tests/__init__.py b/app_python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app_python/tests/test_health.py b/app_python/tests/test_health.py new file mode 100644 index 0000000000..52381e658f --- /dev/null +++ b/app_python/tests/test_health.py @@ -0,0 +1,10 @@ +from app import app + +def test_health_endpoint(): + client = app.test_client() + response = client.get('/health') + + assert response.status_code == 200 + + data = response.get_json() + assert data["status"] == "healthy" \ No newline at end of file diff --git a/app_python/tests/test_root.py b/app_python/tests/test_root.py new file mode 100644 index 0000000000..cdaada74fa --- /dev/null +++ b/app_python/tests/test_root.py @@ -0,0 +1,15 @@ +import json +from app import app + +def test_root_endpoint(): + client = app.test_client() + response = client.get('/') + + assert response.status_code == 200 + + data = response.get_json() + assert isinstance(data, dict) + + assert "service" in data + assert "system" in data + assert "runtime" in data \ No newline at end of file diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000000..7ca3f6c3b4 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,185 @@ +# Task 5 - Documentation + +## 1. Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Kubernetes Cluster │ +│ │ +│ ┌───────────────────────────────────────────────────────────┐ │ +│ │ NodePort Service │ │ +│ │ python-app-service:30080 | │ +│ └───────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌───────────────────────────────────────────────────────────┐ │ +│ │ Deployment │ │ +│ │ python-app │ │ +│ │ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ │ +│ │ │ Pod 1 │ │ Pod 2 │ │ Pod 3 │ │ Pod 4 │ │ │ +│ │ │ :5000 │ │ :5000 │ │ :5000 │ │ :5000 │ │ │ +│ │ └─────────┘ └─────────┘ └─────────┘ └─────────┘ │ │ +│ │ 3-5 replicas │ │ +│ └───────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +**Architecture components:** +- Deployment: Manages 3-5 replicas of application with rolling update strategy +- Service: NodePort type, which opens access on 30080 port +- Pods: Containers with Python Flask app on 5000 port +- Resources: Every pod is limited by 256Mi of memory and 200m CPU + +**Resource Allocation:** +- Memory requests: 128Mi +- Memory limits: 256Mi +- CPU requests: 100m +- CPU limits: 200m + +## 2. Manifest Files +`deployment.yml` key settings: +- 3 replicas: Provides high availability +- Resource requests/limits: Prevents starvation and ensures proper planning +- Liveness probe: Checks /health every 10 seconds, restarts container if error occurs +- Readiness probe: Checks /ready every 5 seconds, removes pod from service service if it's not ready. + +`service.yml` key settings: +- NodePort: Allows access from outside the cluster +- selector: Binds service to pods by label app: python-app +- targetPort: Мapps port 80 service by port 5000 of container + +## 3. Deployment Evidence +Check resources: +![kubectl](screenshots/kubectl%202.png) +![kubectl-service](screenshots/kuberctl%20service%203.png) +![endpoints](screenshots/check%20enpoints%203.png) + +## 4. Operations Performed +**Initial setup**: +![scaling](screenshots/cluster%20setup%201.png) + +**Scaling to 5 Replicas**: +![scaling](screenshots/pods%20scale%204.png) + +**Rolling Update**: +![rolling-update](screenshots/rollout%20update%204.png) + +**Rollback**: +![rollback](screenshots/rollback%204.png) + +**Service Access**: +``` +# Getting service URL +minikube service python-app-service --url + +# Access through browser +open $(minikube service python-app-service --url) +``` + +![check](screenshots/browser%20check.png) + +## 5. Production Considerations +1. What health checks did you implement and why? + - Implemented Health Checks: + + **Liveness Probe (`/health`):** + ```yaml + livenessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + ``` + + **Readiness Probe (/ready):** + ```yaml + readinessProbe: + httpGet: + path: /ready + port: 5000 + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + ``` + + Why this probes: + |Probe | Purpose | Why Implemented | + | ---- | ------- | --------------- | + |Liveness | Determines whether the container needs to be restarted. | Without this, the container with the suspended application would have continued to work, but did not respond to requests. Liveness probe restarts it automatically, providing self-healing. | + |Readiness | Determines whether the pod is ready to receive traffic. | When starting, the application needs time to initialize (download configuration, establish connections). Readiness probe ensures that the pod does not receive traffic until it is fully ready, preventing 5xx errors. | + +2. Resource limits rationale + - **Current сonfiguration** + ```yaml + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + ``` + + - Why these values? + + | Resource | Request | Limit | Rationale | + | -------- | ------- | ----- | --------- | + | Memory | 128Mi | 256Mi | A Flask application with prometheus metrics consumes ~80-100Mi in normal mode. The 128Mi guarantees stable operation even under light load. 256Mi leaves 100% margin for peak loads, garbage collection, and prevents OOM kills. | + | CPU | 100m | 200m | The application does not perform heavy calculations (HTTP processing only). 100m is enough to process ~100-200 requests/sec. The 200m limit prevents the monopolization of the CPU in case of possible problems (for example, an infinite loop or a CPU leak). | + +3. How would you improve this for production? + - **Horizontal Pod Autoscaler (HPA)**. Automatic scaling, depending on CPU/memory metrics + - **Pod Disruption Budget (PDB)**. Guarantees accessibility during voluntary evacuations + - **ConfigMaps for Configuration**. Removing the configuration from the image for flexibility. + +4. Monitoring and observability strategy + - **Metrics collection**. + - `/metrics` endpoint with Prometheus metrics (Counter, Histogram, Gauge) + - Custom metrics + - Grafana Dashboards + - Centralized logging with Loki + + - **Observability Stack Architecture**: + ``` + ┌───────────────────────────────────────────────────────┐ + │ Observability Stack │ + ├───────────────────────────────────────────────────────┤ + │ │ + │ ┌──────────────┐ ┌──────────────┐ │ + │ │ Prometheus │ │ Loki │ │ + │ │ Metrics │ │ Logs │ │ + │ └──────────────┘ └──────────────┘ │ + │ │ │ │ + │ └────────────────|––––––––––––––––––┘ | + │ | │ + │ ┌──────────────┐ │ + │ │ Grafana │ │ + │ │ Dashboards │ │ + │ └──────────────┘ │ + └───────────────────────────────────────────────────────┘ + ``` + +## 6. Challenges & Solutions +### Issue 1: InvalidImageName Error +Problem: Pods failed to start with `InvalidImageName` error + +Solution: Replaced ${{ secrets.DOCKERHUB_USERNAME }} with my actual username + +### Issue 2: CrashLoopBackOff from Missing /ready Endpoint +Problem: Pods were constantly restarting. + +Solution: Added /ready endpoint to app.py, because this endpoint didn't exist + +### Issue 3: Port Mismatch Between Container and Probes +Problem: Probes failed due to port inconsistency + +Solution: Application was listening on port 5000, but probes were configured for port 8000, so that I fixed port in all probes and containerPort + +### What I Learned: +- Kubernetes Requires Explicit Image Names, +- Readiness and Liveness Probes Are Critical +- Port Consistency is Mandatory diff --git a/k8s/deployment.yml b/k8s/deployment.yml new file mode 100644 index 0000000000..fcc565d9b3 --- /dev/null +++ b/k8s/deployment.yml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: python-app + labels: + app: python-app +spec: + replicas: 3 + selector: + matchLabels: + app: python-app + template: + metadata: + labels: + app: python-app + spec: + containers: + - name: python-app + image: gpshfrd/devops-info-python:1.0.3 + ports: + - containerPort: 5000 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + livenessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 5000 + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: python-app-service + labels: + app: python-app +spec: + type: NodePort + selector: + app: python-app + ports: + - protocol: TCP + port: 80 + targetPort: 5000 + nodePort: 30080 \ No newline at end of file diff --git a/k8s/screenshots/browser check.png b/k8s/screenshots/browser check.png new file mode 100644 index 0000000000..bf65df10d5 Binary files /dev/null and b/k8s/screenshots/browser check.png differ diff --git a/k8s/screenshots/check enpoints 3.png b/k8s/screenshots/check enpoints 3.png new file mode 100644 index 0000000000..3b425bef7a Binary files /dev/null and b/k8s/screenshots/check enpoints 3.png differ diff --git a/k8s/screenshots/cluster setup 1.png b/k8s/screenshots/cluster setup 1.png new file mode 100644 index 0000000000..9860ca91b1 Binary files /dev/null and b/k8s/screenshots/cluster setup 1.png differ diff --git a/k8s/screenshots/kubectl 2.png b/k8s/screenshots/kubectl 2.png new file mode 100644 index 0000000000..99b7df88da Binary files /dev/null and b/k8s/screenshots/kubectl 2.png differ diff --git a/k8s/screenshots/kuberctl service 3.png b/k8s/screenshots/kuberctl service 3.png new file mode 100644 index 0000000000..c055d80cf4 Binary files /dev/null and b/k8s/screenshots/kuberctl service 3.png differ diff --git a/k8s/screenshots/pods scale 4.png b/k8s/screenshots/pods scale 4.png new file mode 100644 index 0000000000..24c92c73f2 Binary files /dev/null and b/k8s/screenshots/pods scale 4.png differ diff --git a/k8s/screenshots/rollback 4.png b/k8s/screenshots/rollback 4.png new file mode 100644 index 0000000000..a2c96a3aae Binary files /dev/null and b/k8s/screenshots/rollback 4.png differ diff --git a/k8s/screenshots/rollout update 4.png b/k8s/screenshots/rollout update 4.png new file mode 100644 index 0000000000..f966896b8b Binary files /dev/null and b/k8s/screenshots/rollout update 4.png differ diff --git a/k8s/service.yml b/k8s/service.yml new file mode 100644 index 0000000000..faa2f756dd --- /dev/null +++ b/k8s/service.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: python-app-service + labels: + app: python-app +spec: + type: NodePort + selector: + app: python-app + ports: + - protocol: TCP + port: 80 + targetPort: 5000 + nodePort: 30080 \ No newline at end of file diff --git a/lab1.md b/lab1.md deleted file mode 100644 index 30b74c95f5..0000000000 --- a/lab1.md +++ /dev/null @@ -1,65 +0,0 @@ -# Lab 1: Web Application Development - -## Overview - -In this lab assignment, you will develop a simple web application using Python and best practices. You will also have the opportunity to create a bonus web application using a different programming language. Follow the tasks below to complete the lab assignment. - -## Task 1: Python Web Application - -**6 Points:** - -1. Create `app_python` Folder: - - Create a folder named `app_python` to contain your Python web application files. - - Inside the `app_python` folder, create a file named `PYTHON.md`. - -2. Develop and Test Python Web Application: - - Develop a Python web application that displays the current time in Moscow. - - Choose a suitable framework for your web application and justify your choice in the `PYTHON.md` file. - - Implement best practices in your code and follow coding standards. - - Test your application to ensure the displayed time updates upon page refreshing. - -## Task 2: Well Decorated Description - -**4 Points:** - -1. Update `PYTHON.md`: - - Describe best practices applied in the web application. - - Explain how you followed coding standards, implemented testing, and ensured code quality. - -2. Create `README.md` in `app_python` folder: - - Use a Markdown template to document the Python web application. - -3. Ensure: - - Maintain a clean `.gitignore` file. - - Use a concise `requirements.txt` file for required dependencies. - -### List of Requirements - -- MSK Time timezone set up -- 2 PRs created -- README includes Overview -- Nice Markdown decoration -- Local installation details in README - -## Bonus Task: Additional Web Application - -**2.5 Points:** - -1. Create `app_*` Folder: - - Create a folder named `app_*` in the main project directory, replacing `*` with a programming language of your choice (other than Python). - - Inside the `app_*` folder, create a file named `*`.md. - -2. Develop Your Own Web App: - - Create a web application using the programming language you chose. - - Decide what your web application will display or do, and use your creativity. - -3. Follow Main Task Steps: - - Implement your bonus web application following the same suggestions and steps as the main Python web application task. - -### Guidelines - -- Use proper Markdown formatting and structure for the documentation files. We will use [online one](https://dlaa.me/markdownlint/) to check your `.md` files. -- Organize the files within the lab folder using appropriate naming conventions. -- Create a PR from your fork to the master branch of this repository and from your fork's branch to your fork's master branch with your completed lab assignment. - -> Note: Apply best practices, coding standards, and testing to your Python web application. Explore creativity in your bonus web application, and document your process using Markdown. diff --git a/lab10.md b/lab10.md deleted file mode 100644 index c472086168..0000000000 --- a/lab10.md +++ /dev/null @@ -1,91 +0,0 @@ -# Lab 10: Introduction to Helm - -## Overview - -In this lab, you will become familiar with Helm, set up a local development environment, and generate manifests for your application. - -## Task 1: Helm Setup and Chart Creation - -**6 Points:** - -1. Learn About Helm: - - Begin by exploring the architecture and concepts of Helm: - - [Helm Architecture](https://helm.sh/docs/topics/architecture/) - - [Understanding Helm Charts](https://helm.sh/docs/topics/charts/) - -2. Install Helm: - - Install Helm using the instructions provided: - - [Helm Installation](https://helm.sh/docs/intro/install/) - - [Chart Repository Initialization](https://helm.sh/docs/intro/quickstart/#initialize-a-helm-chart-repository) - -3. Create Your Own Helm Chart: - - Generate a Helm chart for your application. - - Inside the `k8s` folder, create a Helm chart template by using the command `helm create your-app`. - - Replace the default repository and tag inside the `values.yaml` file with your repository name. - - Modify the `containerPort` setting in the `deployment.yml` file. - - If you encounter issues with `livenessProbe` and `readinessProbe`, you can comment them out. - - > For troubleshooting, you can use the `minikube dashboard` command. - -4. Install Your Helm Chart: - - Install your custom Helm chart and ensure that all services are healthy. Verify this by checking the `Workloads` page in the Minikube dashboard. - -5. Access Your Application: - - Confirm that your application is accessible by running the `minikube service your_service_name` command. - -6. Create a HELM.md File: - - Construct a `HELM.md` file and provide the output of the `kubectl get pods,svc` command within it. - -## Task 2: Helm Chart Hooks - -**4 Points:** - -1. Learn About Chart Hooks: - - Familiarize yourself with [Helm Chart Hooks](https://helm.sh/docs/topics/charts_hooks/). - -2. Implement Helm Chart Hooks: - - Develop pre-install and post-install pods within your Helm chart, without adding any complex logic (e.g., use "sleep 20"). You can refer to [Example 1 in the guide](https://www.golinuxcloud.com/kubernetes-helm-hooks-examples/). - -3. Troubleshoot Hooks: - - Execute the following commands to troubleshoot your hooks: - 1. `helm lint ` - 2. `helm install --dry-run helm-hooks ` - 3. `kubectl get po` - -4. Provide Output: - - Execute the following commands and include their output in your report: - 1. `kubectl get po` - 2. `kubectl describe po ` - 3. `kubectl describe po ` - -5. Hook Delete Policy: - - Implement a hook delete policy to remove the hook once it has executed successfully. - -**List of Requirements:** - -- Helm Chart with Hooks implemented, including the hook delete policy. -- Output of the `kubectl get pods,svc` command in `HELM.md`. -- Output of all commands from the step 4 of Task 2 in `HELM.md`. - -## Bonus Task: Helm Library Chart - -**To Earn 2.5 Additional Points:** - -1. Helm Chart for Extra App: - - Prepare a Helm chart for an additional application. - -2. Helm Library Charts: - - Get acquainted with [Helm Library Charts](https://helm.sh/docs/topics/library_charts/). - -3. Create a Library Chart: - - Develop a simple library chart that includes a "labels" template. You can follow the steps outlined in [the Using Library Charts guide](https://austindewey.com/2020/08/17/how-to-reduce-helm-chart-boilerplate-with-library-charts/). Use this library chart for both of your applications. - -### Guidelines - -- Ensure your documentation is clear and well-structured. -- Include all the necessary components. -- Follow appropriate file and folder naming conventions. -- Create and participate in PRs for the peer review process. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Detailed documentation is crucial to ensure that your Helm deployment and hooks function as expected. Engage with the bonus tasks to further enhance your understanding and application deployment skills. diff --git a/lab11.md b/lab11.md deleted file mode 100644 index 4994bb1a80..0000000000 --- a/lab11.md +++ /dev/null @@ -1,85 +0,0 @@ -# Lab 11: Kubernetes Secrets and Hashicorp Vault - -## Overview - -In this lab, you will learn how to manage sensitive data, such as passwords, tokens, or keys, within Kubernetes. Additionally, you will configure CPU and memory limits for your application. - -## Task 1: Kubernetes Secrets and Resource Management - -**6 Points:** - -1. Create a Secret Using `kubectl`: - - Learn about Kubernetes Secrets and create a secret using the `kubectl` command: - - [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) - - [Managing Secrets with kubectl](https://kubernetes.io/docs/tasks/configmap-secret/managing-secret-using-kubectl/#decoding-secret) - -2. Verify and Decode Your Secret: - - Confirm and decode the secret, then create an `11.md` file within the `k8s` folder. Provide the output of the necessary commands inside this file. - -3. Manage Secrets with Helm: - - Use Helm to manage your secrets. - - Create a `secrets.yaml` file in the `templates` folder. - - Define a `secret` object within this YAML file. - - Add an `env` field to your `Deployment`. The path to update is: `spec.template.spec.containers.env`. - - > Refer to this [Helm Secrets Video](https://www.youtube.com/watch?v=hRSlKRvYe1A) for guidance. - - - Update your Helm deployment as instructed in the video. - - Retrieve the list of pods using the command `kubectl get po`. Use the name of the pod as proof of your success within the report. - - Verify your secret inside the pod, for example: `kubectl exec demo-5f898f5f4c-2gpnd -- printenv | grep MY_PASS`. Share this output in `11.md`. - -## Task 2: Vault Secret Management System - -**4 Points:** - -1. Install Vault Using Helm Chart: - - Install Vault using a Helm chart. Follow the steps provided in this guide: - - [Vault Installation Guide](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-sidecar#install-the-vault-helm-chart) - -2. Follow the Tutorial with Your Helm Chart: - - Adapt the tutorial to work with your Helm chart, including the following steps: - - [Set a Secret in Vault](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-sidecar#set-a-secret-in-vault) - - [Configure Kubernetes Authentication](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-sidecar#configure-kubernetes-authentication) - - Be cautious with the service account. If you used `helm create ...`, it will be created automatically. In the guide, they create it manually. - - [Manually Define a Kubernetes Service Account](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-sidecar#define-a-kubernetes-service-account) - -3. Implement Vault Secrets in Your Helm Chart: - - Use the steps from the guide as an example for your Helm chart: - - [Update values.yaml](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-sidecar#launch-an-application) - - [Add Labels](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-sidecar#inject-secrets-into-the-pod) - - Test to ensure your credentials are injected successfully. Use the `kubectl exec -it -- bash` command to access the container. Verify the injected secrets using `cat /path/to/your/secret` and `df -h`. Share the output in the `11.md` report. - - Apply a template as described in the guide. Test the updates as you did in the previous step and provide the outputs in `11.md`. - -**List of Requirements:** - -- Proof of work with a secret in `11.md` for the Task 1 - steps 2 and 3. -- `secrets.yaml` file. -- Resource requests and limits for CPU and memory. -- Vault configuration implemented, with proofs in `11.md`. - -## Bonus Task: Resource Management and Environment Variables - -**2.5 Points:** - -1. Read About Resource Management: - - Familiarize yourself with resource management in Kubernetes: - - [Resource Management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) - -2. Set Up Requests and Limits for CPU and Memory for Both Helm Charts: - - Configure resource requests and limits for CPU and memory for your application. - - Test to ensure these configurations work correctly. - -3. Add Environment Variables for Your Containers for Both Helm Charts: - - Read about Kubernetes environment variables: - - [Kubernetes Environment Variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) - - Update your Helm chart with several environment variables using named templates. Move these variables to the `_helpers.tpl` file: - - [Helm Named Templates](https://helm.sh/docs/chart_template_guide/named_templates/) - -### Guidelines - -- Ensure that your documentation is clear and organized. -- Include all the necessary components. -- Follow appropriate file and folder naming conventions. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Thorough documentation is essential to demonstrate your success in managing secrets and resource allocation in Kubernetes. Explore the bonus tasks to enhance your skills further. diff --git a/lab12.md b/lab12.md deleted file mode 100644 index efb72a29ec..0000000000 --- a/lab12.md +++ /dev/null @@ -1,68 +0,0 @@ -# Lab 12: Kubernetes ConfigMaps - -## Overview - -In this lab, you'll delve into Kubernetes ConfigMaps, focusing on managing non-confidential data and upgrading your application for persistence. ConfigMaps provide a way to decouple configuration artifacts from image content, allowing you to manage configuration data separately from the application. - -## Task 1: Upgrade Application for Persistence - -**6 Points:** - -1. Upgrade Your Application: - - Modify your application to: - - Implement a counter logic in your application to keep track of the number of times it's accessed. - - Save the counter number in the `visits` file. - - Introduce a new endpoint `/visits` to display the recorded visits. - - Test the changes: - - Update your `docker-compose.yml` to include a new volume with your `visits` file. - - Verify that the enhancements work as expected, you must see the updated number in the `visits` file on the host machine. - - Update the `README.md` for your application. - -## Task 2: ConfigMap Implementation - -**4 Points:** - -1. Understand ConfigMaps: - - Read about ConfigMaps in Kubernetes: - - [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/) - -2. Mount a Config File: - - Create a `files` folder with a `config.json` file. - - Populate `config.json` with data in JSON format. - - Use Helm to mount `config.json`: - - Create a `configMap` manifest, extracting data from `config.json` using `.Files.Get`. - - Update `deployment.yaml` with `Volumes` and `VolumeMounts`. - - [Example](https://carlos.mendible.com/2019/02/10/kubernetes-mount-file-pod-with-configmap/) - - Install the updated Helm chart and verify success: - - Retrieve the list of pods: `kubectl get po`. - - Use the pod name as proof of successful deployment. - - Check the ConfigMap inside the pod, e.g., `kubectl exec demo-758cc4d7c4-cxnrn -- cat /config.json`. - -3. Documentation: - - Create `12.md` in the `k8s` folder and include the output of relevant commands. - -**List of Requirements:** - -- `config.json` in the `files` folder. -- `configMap` retrieving data from `config.json` using `.Files.Get`. -- `Volume`s and `VolumeMount`s in `deployments.yml`. -- `12.md` documenting the results of commands. - -## Bonus Task: ConfigMap via Environment Variables - -**2.5 Points:** - -1. Upgrade Bonus App: - - Implement persistence logic in your bonus app. - -2. ConfigMap via Environment Variables: - - Utilize ConfigMap via environment variables in a running container using the `envFrom` property. - - Provide proof with the output of the `env` command inside your container. - -### Guidelines - -- Maintain clear and organized documentation. -- Use appropriate naming conventions for files and folders. -- For your repository PR, ensure it's from the `lab12` branch to the main branch. - -> Note: Clear documentation is crucial to demonstrate successful data persistence and ConfigMap utilization in Kubernetes. Explore the bonus tasks to further enhance your skills. diff --git a/lab13.md b/lab13.md deleted file mode 100644 index e6f6c919f8..0000000000 --- a/lab13.md +++ /dev/null @@ -1,212 +0,0 @@ -# Lab 13: ArgoCD for GitOps Deployment - -## Overview - -In this lab, you will implement ArgoCD to automate Kubernetes application deployments using GitOps principles. You’ll install ArgoCD via Helm, configure it to manage your Python app, and simulate production-like workflows. - -## Task 1: Deploy and Configure ArgoCD - -**6 Points:** - -1. Install ArgoCD via Helm - - Add the ArgoCD Helm repository: - - ```bash - helm repo add argo https://argoproj.github.io/argo-helm - ``` - - [ArgoCD Helm Chart Docs](https://github.com/argoproj/argo-helm) - - - Install ArgoCD: - - ```bash - helm install argo argo/argo-cd --namespace argocd --create-namespace - ``` - - [ArgoCD Installation Guide](https://argo-cd.readthedocs.io/en/stable/getting_started/) - - - Verify installation: - - ```bash - kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=argocd-server -n argocd --timeout=90s - ``` - -2. Install ArgoCD CLI - - Install the ArgoCD CLI tool (required for command-line interactions): - - ```bash - # For macOS (Homebrew): - brew install argocd - - # For Debian/Ubuntu: - sudo apt-get install -y argocd - - # For other OS/architectures: - curl -sSL -o argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64 - chmod +x argocd - sudo mv argocd /usr/local/bin/ - ``` - - [ArgoCD CLI Docs](https://argo-cd.readthedocs.io/en/stable/cli_installation/) - - - Verify CLI installation: - - ```bash - argocd version - ``` - -3. Access the ArgoCD UI - - Forward the ArgoCD server port: - - ```bash - kubectl port-forward svc/argocd-server -n argocd 8080:443 & - ``` - - - Log in using the initial admin password: - - ```bash - # Retrieve the password: - kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 --decode - - # Log in via CLI: - argocd login localhost:8080 --insecure - argocd account login - ``` - - [ArgoCD Authentication Docs](https://argo-cd.readthedocs.io/en/stable/user-guide/accessing/) - -4. Configure Python App Sync - - Create an ArgoCD folder: - Add an `ArgoCD` folder in your `k8s` directory for ArgoCD manifests. - - - Define the ArgoCD Application: - Create `argocd-python-app.yaml` in the `ArgoCD` folder: - - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Application - metadata: - name: python-app - namespace: argocd - spec: - project: default - source: - repoURL: https://github.com//S25-core-course-labs.git - targetRevision: lab13 - path: - helm: - valueFiles: - - values.yaml - destination: - server: https://kubernetes.default.svc - namespace: default - syncPolicy: - automated: {} - ``` - - [ArgoCD Application Manifest Docs](https://argo-cd.readthedocs.io/en/stable/operator-manual/declarative_setup/) - - - Apply the configuration: - - ```bash - kubectl apply -f ArgoCD/argocd-python-app.yaml - ``` - - - Verify sync: - - ```bash - argocd app sync python-app - argocd app status python-app - ``` - -5. Test Sync Workflow - - Modify `values.yaml` (e.g., update `replicaCount`). - - Commit and push changes to the target branch from the config. - - Observe ArgoCD auto-sync the update: - - ```bash - argocd app status python-app - ``` - -### Task 2: Multi-Environment Deployment & Auto-Sync - -**4 Points:** - -1. Set Up Multi-Environment Configurations - - Extend your Python app’s Helm chart to support `dev` and `prod` environments. - - Create environment-specific values files (`values-dev.yaml`, `values-prod.yaml`). - -2. Create Namespaces - - ```bash - kubectl create namespace dev - kubectl create namespace prod - ``` - -3. Deploy Multi-Environment via ArgoCD - - Define two ArgoCD applications with auto-sync: - `argocd-python-dev.yaml` and `argocd-python-prod.yaml` (as before). - -4. Enable Auto-Sync - - Test auto-sync by updating `values-prod.yaml` and pushing to Git. - -5. Self-Heal Testing - - Test 1: Manual Override of Replica Count - 1. Modify the deployment’s replica count manually: - - ```bash - kubectl patch deployment python-app-prod -n prod --patch '{"spec":{"replicas": 3}}' - ``` - - 2. Observe ArgoCD auto-revert the change (due to `syncPolicy.automated`): - - ```bash - argocd app sync python-app-prod - argocd app status python-app-prod - ``` - - - Test 2: Delete a Pod (Replica) - 1. Delete a pod in the `prod` namespace: - - ```bash - kubectl delete pod -n prod -l - ``` - - 2. Verify Kubernetes recreates the pod to match the deployment’s `replicaCount`: - - ```bash - kubectl get pods -n prod -w - ``` - - 3. Confirm ArgoCD shows no drift (since pod deletions don’t affect the desired state): - - ```bash - argocd app diff python-app-prod - ``` - -6. Documentation - - In `13.md`, include: - - Output of `kubectl get pods -n prod` before and after pod deletion. - - Screenshots of ArgoCD UI showing sync status and the dashboard after both tests. - - Explanation of how ArgoCD handles configuration drift vs. runtime events. - -## Bonus Task: Sync Your Bonus App with ArgoCD - -**2.5 Points:** - -1. Configure ArgoCD for Bonus App - - Create an `argocd--app.yaml` similar to Task 1, pointing to your bonus app’s helm chart folder. - - Sync and validate deployment with: - - ```bash - kubectl get pods -n - ``` - -### Guidelines - -- Follow the [ArgoCD docs](https://argo-cd.readthedocs.io/) for advanced configurations. -- Use consistent naming conventions (e.g., `lab13` branch for Git commits). -- Document all steps in `13.md` (include diffs, outputs, and UI screenshots). -- For your repository PR, ensure it's from the `lab14` branch to the main branch. - -> **Note**: This lab emphasizes GitOps workflows, environment isolation, and automation. Mastery of ArgoCD will streamline your CI/CD pipelines in real-world scenarios. diff --git a/lab14.md b/lab14.md deleted file mode 100644 index d1d6ba51cd..0000000000 --- a/lab14.md +++ /dev/null @@ -1,106 +0,0 @@ -# Lab 14: Kubernetes StatefulSet - -## Overview - -In this lab, you'll explore Kubernetes StatefulSets, focusing on managing stateful applications with guarantees about the ordering and uniqueness of a set of Pods. - -## Task 1: Implement StatefulSet in Helm Chart - -**6 Points:** - -1. Understand StatefulSets: - - Read about StatefulSet objects: - - [Concept](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) - - [Tutorial](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/) - -2. Update Helm Chart: - - Rename `deployment.yml` to `statefulset.yml`. - - Create a manifest for StatefulSet following the tutorial. - - Test with command: `helm install --dry-run --debug name_of_your_chart path_to_your_chart`. - - Fix any issues and deploy it. - - Apply best practices by moving values to variables in `values.yml` meaningfully. - -## Task 2: StatefulSet Exploration and Optimization - -**4 Points:** - -1. Research and Documentation: - - Create `14.md` report. - - Include the output of `kubectl get po,sts,svc,pvc` commands. - - Use `minikube service name_of_your_statefulset` command to access your app. - - Access the root path of your app from different tabs and modes in your browser. - - Check the content of your file in each pod, e.g., `kubectl exec pod/demo-0 -- cat visits`, and provide the output for all replicas. - - Describe and explain differences in the report. - -2. Persistent Storage Validation - - Delete a pod: - - ```bash - kubectl delete pod app-stateful-0 - ``` - - - Verify that the PVC and data persist: - - ```bash - kubectl get pvc - kubectl exec app-stateful-0 -- cat /data/visits - ``` - -3. Headless Service Access - - Access pods via DNS: - - ```bash - kubectl exec app-stateful-0 -- nslookup app-stateful-1.app-stateful - ``` - - - Document DNS resolution in `14.md`. - -4. Monitoring & Alerts - - Add liveness/readiness probes to your StatefulSet. - - Describe in `14.md`: - - How probes ensure pod health. - - Why they’re critical for stateful apps. - -5. Ordering Guarantee and Parallel Operations: - - Explain why ordering guarantees are unnecessary for your app. - - Implement a way to instruct the StatefulSet controller to launch or terminate all Pods in parallel. - -**List of Requirements:** - -- Outputs of commands in `14.md`. -- Results of the "number of visits" command for each pod, with an explanation in `14.md`. -- Answers to questions in point 2 of `14.md`. -- Implementation of parallel launch and terminate. - -## Bonus Task: Update Strategies - -**2.5 Points:** - -1. Apply StatefulSet to Bonus App - - Convert your bonus app’s Helm chart to use a StatefulSet. - -2. Explore Update Strategies - - Implement Rolling Updates: - - ```yaml - spec: - updateStrategy: - type: RollingUpdate - rollingUpdate: - partition: 1 - ``` - - - Test Canaries: - Update a subset of pods first. - - - Document in `14.md`: - - Explain `OnDelete`, `RollingUpdate`, and their use cases. - - Compare with Deployment update strategies. - -### Guidelines - -- Maintain clear and organized documentation. -- Use appropriate naming conventions for files and folders. -- For your repository PR, ensure it's from the `lab14` branch to the main branch. - -> Note: Understanding StatefulSets and their optimization is crucial for managing stateful applications in Kubernetes. Explore the bonus tasks to further enhance your skills. diff --git a/lab15.md b/lab15.md deleted file mode 100644 index 887587145d..0000000000 --- a/lab15.md +++ /dev/null @@ -1,78 +0,0 @@ -# Lab 15: Kubernetes Monitoring and Init Containers - -## Overview - -In this lab, you will explore Kubernetes cluster monitoring using Prometheus with the Kube Prometheus Stack. Additionally, you'll delve into the concept of Init Containers in Kubernetes. - -## Task 1: Kubernetes Cluster Monitoring with Prometheus - -**6 Points:** - -1. This lab was tested on a specific version of components: - - Minikube v1.33.0 - - Minikube kubectl v1.28.3 - - kube-prometheus-stack-57.2.0 v0.72.0 - - the minikube start command - `minikube start --driver=docker --container-runtime=containerd` - -2. Read about `Kube Prometheus Stack`: - - [Helm chart with installation guide](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) - - [Explanation of components](https://github.com/prometheus-operator/kube-prometheus#kubeprometheus) - -3. Describe Components: - - Create `15.md` and detail the components of the Kube Prometheus Stack, explaining their roles and functions. Avoid direct copy-pasting; provide a personal understanding. - -4. Install Helm Charts: - - Install the Kube Prometheus Stack to your Kubernetes cluster. - - Install your app's Helm chart. - - Provide the output of the `kubectl get po,sts,svc,pvc,cm` command in the report and explain each part. - -5. Utilize Grafana Dashboards: - - Access Grafana using `minikube service monitoring-grafana`. - - Explore existing dashboards to find information about your cluster: - 1. Check CPU and Memory consumption of your StatefulSet. - 2. Identify Pods with higher and lower CPU usage in the default namespace. - 3. Monitor node memory usage in percentage and megabytes. - 4. Count the number of pods and containers managed by the Kubelet service. - 5. Evaluate network usage of Pods in the default namespace. - 6. Determine the number of active alerts; also check the Web UI with `minikube service monitoring-kube-prometheus-alertmanager`. - - Provide answers to all these points in the report. - -## Task 2: Init Containers - -**4 Points:** - -1. Read about `Init Containers`: - - [Concept](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) - - [Tutorial](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-initialization/#create-a-pod-that-has-an-init-container) - -2. Implement Init Container: - - Create a new Volume. - - Implement an Init container to download any file using `wget` (you can use a site from the example). - - Provide proof of success, e.g., `kubectl exec pod/demo-0 -- cat /test.html`. - -**List of Requirements:** - -- Detailed explanation of monitoring stack components in `15.md`. -- Output and explanation of `kubectl get po,sts,svc,pvc,cm`. -- Answers to all 6 questions from point 4 in `15.md`. -- Implementation of Init Container. -- Proof of Init Container downloading a file. - -## Bonus Task: App Metrics & Multiple Init Containers - -**2.5 Points:** - -1. App Metrics: - - Fetch metrics from your app and provide proof. - -2. Init Container Queue: - - Create a queue of three Init containers, with any logic like adding new lines to the same file. - - Provide proof using the `cat` tool. - -### Guidelines - -- Ensure clear and organized documentation. -- Use appropriate naming conventions for files and folders. -- For your repository PR, ensure it's from the `lab15` branch to the main branch. - -> Note: Demonstrate successful implementation and understanding of Kubernetes monitoring and Init Containers. Take your time to explore the bonus tasks for additional learning opportunities. diff --git a/lab16.md b/lab16.md deleted file mode 100644 index 37912fc50b..0000000000 --- a/lab16.md +++ /dev/null @@ -1,75 +0,0 @@ -# Lab 16: IPFS and Fleek - -In this lab, you will explore essential DevOps tools and set up a project on the Fleek service. Follow the tasks below to complete the lab assignment. - -## Task 1: Set Up an IPFS Gateway Using Docker - -Objective: Understand and implement an IPFS gateway using Docker, upload a file, and verify it via an IPFS cluster. - -1. Set Up IPFS Gateway: - - Install Docker on your machine if it's not already installed. - - [Docker Installation Guide](https://docs.docker.com/get-docker/) - - - Pull the IPFS Docker image and run an IPFS container: - - ```sh - docker pull ipfs/go-ipfs - docker run -d --name ipfs_host -v /path/to/folder/with/file:/export -v ipfs_data:/data/ipfs -p 8080:8080 -p 4001:4001 -p 5001:5001 ipfs/go-ipfs - ``` - - - Verify the IPFS container is running: - - ```sh - docker ps - ``` - -2. Upload a File to IPFS: - - Open a browser and access the IPFS web UI: - - ```sh - http://127.0.0.1:5001/webui/ - ``` - - - Explore the web UI and wait for 5 minutes to sync up with the network. - - Upload any file via the web UI. - - Use the obtained hash to access the file via any public IPFS gateway. Here are a few options: - - [IPFS.io Gateway](https://ipfs.io/ipfs/) - - [Cloudflare IPFS Gateway](https://cloudflare-ipfs.com/ipfs/) - - [Infura IPFS Gateway](https://ipfs.infura.io/ipfs/) - - - Append your file hash to any of the gateway URLs to verify your file is accessible. Note that it may fail due to network overload, so don't worry if you can't reach it. - -3. Documentation: - - Create a `submission2.md` file. - - Share information about connected peers and bandwidth in your report. - - Provide the hash and the URLs used to verify the file on the IPFS gateways. - -## Task 2: Set Up Project on Fleek.xyz - -Objective: Set up a project on the Fleek service and share the IPFS link. - -1. Research: - - Understand what IPFS is and its purpose. - - Explore Fleek's features. - -2. Set Up: - - Sign up for a Fleek account if you haven't already. - - Use your fork of the Labs repository as your project source. Optionally, set up your own website (notify us in advance). - - Configure the project settings on Fleek. - - Deploy the Labs repository to Fleek, ensuring it is uploaded to IPFS. - -3. Documentation: - - Share the IPFS link and domain of the deployed project in the `submission2.md` file. - -## Additional Resources - -- [IPFS Documentation](https://docs.ipfs.io/) -- [Fleek Documentation](https://docs.fleek.xyz/) - -### Guidelines - -- Use proper Markdown formatting for documentation files. -- Organize files with appropriate naming conventions. -- Create a Pull Request to the main branch of the repository with your completed lab assignment. - -> Note: Actively explore and document your findings to gain hands-on experience with IPFS and Fleek. diff --git a/lab16/index.html b/lab16/index.html deleted file mode 100644 index acce39eee3..0000000000 --- a/lab16/index.html +++ /dev/null @@ -1,303 +0,0 @@ - - - - - - DevOps Engineering Expert Track - - - - -
-
- -
-
- -
-
-

Master Modern DevOps Practices

-

16 hands-on labs covering Kubernetes, Terraform, CI/CD, and more

- Start Free Trial → -
-
- -
-

Why This Course?

-
-
- -

16 Advanced Labs

-

Build production-ready systems from scratch

-
-
- -

Industry-Standard Tools

-

Terraform, ArgoCD, Prometheus, Vault, and more

-
-
- -

Job-Ready Skills

-

Learn tools used by top tech companies

-
-
-
- -
-
-

Lab Syllabus (2025 Edition)

-
    -
  1. Lab 1: Web Application Development
    2. -
  2. Lab 2: Containerization
    3. -
  3. Lab 3: Continuous Integration
    4. -
  4. Lab 4: Infrastructure as Code & Terraform
    5. -
  5. Lab 5: Configuration Management
    6. -
  6. Lab 6: Ansible Automation
    7. -
  7. Lab 7: Observability, Logging, Loki Stack
    8. -
  8. Lab 8: Monitoring & Prometheus
    9. -
  9. Lab 9: Kubernetes & Declarative Manifests
    10. -
  10. Lab 10: Helm Charts & Library Charts
    11. -
  11. Lab 11: Kubernetes Secrets Management (Vault, ConfigMaps)
    12. -
  12. Lab 12: Kubernetes ConfigMaps & Environment Variables
    13. -
  13. Lab 13: GitOps with ArgoCD
    14. -
  14. Lab 14: StatefulSet Optimization
    15. -
  15. Lab 15: Kubernetes Monitoring & Init Containers
    16. -
  16. Lab 16: IPFS & Fleek Decentralization
    17. -
-
-
- -
-

Learning Progression

-
-

Phase 1: Foundations (Labs 1-6)

-

Web Dev → Containers → CI/CD → IaC → Ansible

-
-
-

Phase 2: Observability (Labs 7-8)

-

Logging → Monitoring → Loki/Prometheus

-
-
-

Phase 3: Kubernetes Mastery (Labs 9-12)

-

Deployments → Helm → Secrets → ConfigMaps

-
-
-

Phase 4: Expert Track (Labs 13-16)

-

GitOps → StatefulSets → IPFS → Final Project

-
-
- - - - - - \ No newline at end of file diff --git a/lab2.md b/lab2.md deleted file mode 100644 index ff71bc227d..0000000000 --- a/lab2.md +++ /dev/null @@ -1,85 +0,0 @@ -# Lab 2: Containerization - Docker - -## Overview - -In this lab assignment, you will learn to containerize applications using Docker, while focusing on best practices. Additionally, you will explore Docker multi-stage builds. Follow the tasks below to complete the lab assignment. - -## Task 1: Dockerize Your Application - -**6 Points:** - -1. Create a `Dockerfile`: - - Inside the `app_python` folder, craft a `Dockerfile` for your application. - - Research and implement Docker best practices. Utilize a Dockerfile linter for quality assurance. - -2. Build and Test Docker Image: - - Build a Docker image using your Dockerfile. - - Thoroughly test the image to ensure it functions correctly. - -3. Push Image to Docker Hub: - - If you lack a public Docker Hub account, create one. - - Push your Docker image to your public Docker Hub account. - -4. Run and Verify Docker Image: - - Retrieve the Docker image from your Docker Hub account. - - Execute the image and validate its functionality. - -## Task 2: Docker Best Practices - -**4 Points:** - -1. Enhance your docker image by implementing [Docker Best Practices](https://docs.docker.com/build/building/best-practices/). - - No root user inside, or you will get no points at all. - -2. Write `DOCKER.md`: - - Inside the `app_python` folder, create a `DOCKER.md` file. - - Elaborate on the best practices you employed within your Dockerfile. - - Implementing and listing numerous Docker best practices will earn you more points. - -3. Enhance the README.md: - - Update the `README.md` file in the `app_python` folder. - - Include a dedicated `Docker` section, explaining your containerized application and providing clear instructions for execution. - - How to build? - - How to pull? - - How to run? - -### List of Requirements - -- Rootless container. -- Use COPY, but only specific files. -- Layer sanity. -- Use `.dockerignore`. -- Use a precise version of your base image and language, example `python:3-alpine3.15`. - -## Bonus Task: Multi-Stage Builds Exploration - -**2.5 Points:** - -1. Dockerize Previous App: - - Craft a `Dockerfile` for the application from the prior lab. - - Place this Dockerfile within the corresponding `app_*` folder. - -2. Follow Main Task Guidelines: - - Apply the same steps and suggestions as in the primary Dockerization task. - -3. Study Docker Multi-Stage Builds: - - Familiarize yourself with Docker multi-stage builds. - - Consider implementing multi-stage builds, only if they enhance your project's structure and efficiency. - -4. Study Distroless Images: - - Explore how to use Distroless images by reviewing the official documentation: [GoogleContainerTools/distroless](https://github.com/GoogleContainerTools/distroless). - - Create new `distroless.Dockerfile` files for your Python app and your second app. - - Use the `nonroot` tag for both images to ensure they run with non-root privileges. - - Verify that the applications work correctly with the Distroless images. - - Compare the sizes of your previous Docker images with the new Distroless-based images. - - In the `DOCKER.md` file, describe the differences between the Distroless images and your previous images. Explain why these differences exist (e.g., smaller size, reduced attack surface, etc.). - - Include a screenshot of your final results (e.g., image sizes). - - Add a new section to the `README.md` file titled "Distroless Image Version". - -### Guidelines - -- Utilize appropriate Markdown formatting and structure for all documentation. -- Organize files within the lab folder with suitable naming conventions. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Utilize Docker to containerize your application, adhering to best practices. Explore Docker multi-stage builds for a deeper understanding, and document your process using Markdown. diff --git a/lab3.md b/lab3.md deleted file mode 100644 index 2f4899750a..0000000000 --- a/lab3.md +++ /dev/null @@ -1,53 +0,0 @@ -# Lab 3: Continuous Integration Lab - -## Overview - -In this lab assignment, you will delve into continuous integration (CI) practices by focusing on code testing, setting up Git Actions CI, and optimizing workflows. Additionally, you will have the opportunity to explore bonus tasks to enhance your CI knowledge. Follow the tasks below to complete the lab assignment. - -## Task 1: Code Testing and Git Actions CI - -**6 Points:** - -1. Code Testing: - - Begin by researching and implementing best practices for code testing. - - Write comprehensive unit tests for your application. - - In the `PYTHON.md` file, describe the unit tests you've created and the best practices you applied. - - Enhance the `README.md` file by adding a "Unit Tests" section. - -2. Set Up Git Actions CI: - - Create a CI workflow using GitHub Actions to build and test your Python project. Refer to the [official GitHub Actions documentation](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python) for guidance. - - Ensure your CI workflow includes at least three essential steps: Dependencies, Linter, and Tests. - - Integrate Docker-related steps into your CI workflow, at least two steps Login, Build & Push. You can refer to the [Docker GitHub Actions documentation](https://docs.docker.com/ci-cd/github-actions/) for assistance. - - Update the `README.md` file to provide information about your CI workflow. - -## Task 2: CI Workflow Improvements - -**4 Points:** - -1. Workflow Enhancements: - - Add a workflow status badge to your repository for visibility. - - Dive into best practices for CI workflows and apply them to optimize your existing workflow. - - Utilize build cache to enhance workflow efficiency. - - Create a `CI.md` file and document the best practices you've implemented. - -2. Implement Snyk Vulnerability Checks: - - Integrate Snyk into your CI workflow to identify and address vulnerabilities in your projects. You can refer to the [Python example](https://github.com/snyk/actions/tree/master/python-3.8) for guidance, check [another option](https://docs.snyk.io/integrations/snyk-ci-cd-integrations/github-actions-integration#use-your-own-development-environment) how to install dependencies if you face any issue. - -## Bonus Task - -**2.5 Points:** - -1. Follow the Main Task Steps: - - Apply the same steps as in the primary CI task to set up CI workflows for an extra application. You can find useful examples in the [GitHub Actions starter workflows](https://github.com/actions/starter-workflows/tree/main/ci). - -2. CI Workflow Improvements: - 1. Python App CI: Configure the CI workflow to run only when changes occur in the `app_python` folder. - 2. Extra Language App CI: Configure the CI workflow to run only when changes occur in the `app_` folder. - -### Guidelines - -- Use proper Markdown formatting and structure for all documentation files. -- Organize files within the lab folder with suitable naming conventions. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Implement CI best practices, optimize your workflows, and explore bonus tasks to deepen your understanding of continuous integration. diff --git a/lab4.md b/lab4.md deleted file mode 100644 index e88b5e63e5..0000000000 --- a/lab4.md +++ /dev/null @@ -1,84 +0,0 @@ -# Lab 4: Infrastructure as Code Lab - -## Overview - -In this lab assignment, you will explore Infrastructure as Code (IAC) using Terraform. You'll build Docker and AWS infrastructures and dive into managing GitHub repositories through Terraform. Additionally, there are bonus tasks to enhance your Terraform skills. Follow the tasks below to complete the lab assignment. - -## Task 1: Introduction to Terraform - -**6 Points:** - -0. You will need a VPN tool for this lab - -1. Get Familiar with Terraform: - - Begin by familiarizing yourself with Terraform by reading the [introduction](https://www.terraform.io/intro/index.html) and exploring [best practices](https://www.terraform.io/docs/cloud/guides/recommended-practices/index.html). - -2. Set Up Terraform Workspace: - - Create a `terraform` folder to organize your Terraform workspaces. - - Inside the `terraform` folder, create a file named `TF.md`. - -3. Docker Infrastructure Using Terraform: - - Follow the [Docker tutorial](https://learn.hashicorp.com/collections/terraform/docker-get-started) for building a Docker infrastructure with Terraform. - - Perform the following tasks as instructed in the tutorial: - - Install Terraform. - - Build the Infrastructure. - - Provide the output of the following commands in the `TF.md` file: - - ```sh - terraform state show - terraform state list - ``` - - - Document a part of the log with the applied changes. - - Utilize input variables to rename your Docker container. - - Finish the tutorial and provide the output of the `terraform output` command in the `TF.md` file. - -4. Yandex Cloud Infrastracture Using Terraform: - - Create an account on [Yandex Cloud](https://cloud.yandex.com/). - - Check for available free-tier options and select a free VM instance suitable for this lab. - - Follow the [Yandex Quickstart Guide](https://yandex.cloud/en-ru/docs/tutorials/infrastructure-management/terraform-quickstart#linux_1) to set up and configure Terraform for managing Yandex Cloud resources. - - Document the entire process, including setup steps, configurations, and any challenges encountered, in the `TF.md` file. - -5. [Optioinal] AWS Infrastructure Using Terraform: - - Follow the [AWS tutorial](https://learn.hashicorp.com/tutorials/terraform/aws-build?in=terraform/aws-get-started) alongside the instructions from the previous step. - -## Task 2: Terraform for GitHub - -**4 Points:** - -1. GitHub Infrastructure Using Terraform: - - Utilize the [GitHub provider for Terraform](https://registry.terraform.io/providers/integrations/github/latest/docs). - - Create a directory inside the `terraform` folder specifically for managing your GitHub project infrastructure. - - Build GitHub infrastructure following a reference like [this example](https://dev.to/pwd9000/manage-and-maintain-github-with-terraform-2k86). Prepare `.tf` files that include: - - Repository name - - Repository description - - Visibility settings - - Default branch - - Branch protection rule for the default branch - - Avoid placing your token as a variable in the code; instead, use an environment variable. - -2. Import Existing Repository: - - Use the `terraform import` command to import your current GitHub repository into your Terraform configuration. No need to create a new one. Example: `terraform import "github_repository.core-course-labs" "core-course-labs"`. - -3. Apply Terraform Changes: - - Apply changes from your Terraform configuration to your GitHub repository. - -4. Document Best Practices: - - Provide Terraform-related best practices that you applied in the `TF.md` file. - -## Bonus Task: Adding Teams - -**2.5 Points:** - -1. GitHub Teams Using Terraform: - - You need to create a new organization. - - Extend your Terraform configuration to add several teams to your GitHub repository, each with different levels of access. - - Apply the changes and ensure they take effect in your GitHub repository. - -### Guidelines - -- Use proper Markdown formatting and structure for documentation files. -- Organize files within the lab folder with suitable naming conventions. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Dive into Terraform to manage infrastructures efficiently. Explore the AWS and Docker tutorials, and don't forget to document your process and best practices in the `TF.md` file. diff --git a/lab5.md b/lab5.md deleted file mode 100644 index bead18ceea..0000000000 --- a/lab5.md +++ /dev/null @@ -1,141 +0,0 @@ -# Lab 5: Ansible and Docker Deployment - -## Overview - -In this lab, you will get acquainted with Ansible, a powerful configuration management and automation tool. Your objective is to use Ansible to deploy Docker on a newly created cloud VM. This knowledge will be essential for your application deployment in the next lab. - -## Task 1: Initial Setup - -**6 Points:** - -1. Repository Structure: - - Organize your repository following the recommended structure below: - - ```sh - . - |-- README.md - |-- ansible - | |-- inventory - | | `-- default_aws_ec2.yml - | |-- playbooks - | | `-- dev - | | `-- main.yaml - | |-- roles - | | |-- docker - | | | |-- defaults - | | | | `-- main.yml - | | | |-- handlers - | | | | `-- main.yml - | | | |-- tasks - | | | | |-- install_compose.yml - | | | | |-- install_docker.yml - | | | | `-- main.yml - | | | `-- README.md - | | `-- web_app - | | |-- defaults - | | | `-- main.yml - | | |-- handlers - | | | `-- main.yml - | | |-- meta - | | | `-- main.yml - | | |-- tasks - | | | `-- main.yml - | | `-- templates - | | `-- docker-compose.yml.j2 - | `-- ansible.cfg - |-- app_go - |-- app_python - `-- terraform - ``` - -2. Installation and Introduction: - - Install Ansible and familiarize yourself with its basics. You can follow the [Ansible installation guide](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html). - -3. Use an Existing Ansible Role for Docker: - - Utilize an existing Ansible role for Docker from `ansible-galaxy` as a template. You can explore [this Docker role](https://github.com/geerlingguy/ansible-role-docker) as an example. - -4. Create a Playbook and Testing: - - Develop an Ansible playbook for deploying Docker. - - Test your playbook to ensure it works as expected. - -## Task 2: Custom Docker Role - -**4 Points:** - -1. Create Your Custom Docker Role: - - Develop a custom Ansible role for Docker with the following tasks: - 1. Install Docker and Docker Compose. - 2. Update your playbook to utilize this custom role. [Tricks and Tips](https://docs.ansible.com/ansible/latest/user_guide/playbooks_best_practices.html). - 3. Test your playbook with the custom role to ensure successful deployment. - 4. Make sure the role has a task to configure Docker to start on boot (`systemctl enable docker`). - 5. Include a task to add the current user to the `docker` group to avoid using `sudo` for Docker commands. - -2. Documentation: - - Develop an `ANSIBLE.md` file in the `ansible` folder to document your Ansible-related work. - - Create a `README.md` file in the `ansible/roles/docker` folder. - - Use a Markdown template to describe your Docker role, its requirements and usage. - - Example `README.md` template for the Docker role: - - ```markdown - # Docker Role - - This role installs and configures Docker and Docker Compose. - - ## Requirements - - - Ansible 2.9+ - - Ubuntu 22.04 - - ## Role Variables - - - `docker_version`: The version of Docker to install (default: `latest`). - - `docker_compose_version`: The version of Docker Compose to install (default: `1.29.2`). - - ## Example Playbook - - ```yaml - - hosts: all - roles: - - role: docker - ``` - -3. Deployment Output: - - Execute your playbook to deploy the Docker role. - - Provide the last 50 lines of the output from your deployment command in the `ANSIBLE.md` file. - - Use the `--check` flag with `ansible-playbook` to perform a dry run and verify changes before applying them. - - Example command: - - ```sh - ansible-playbook --diff - ``` - -4. **Inventory Details:** - - Execute the following command `ansible-inventory -i .yaml --list` and provide its output in the `ANSIBLE.md` file. - - Validate the inventory file using `ansible-inventory -i .yaml --graph` to visualize the inventory structure. - - Ensure you have documented the inventory information. - -## Bonus Task: Dynamic Inventory - -**2.5 Points:** - -1. Set up Dynamic Inventory: - - Implement dynamic inventory for your cloud environment, if available. - - You may explore ready-made solutions for dynamic inventories: - - - [AWS Example](https://docs.ansible.com/ansible/latest/collections/amazon/aws/aws_ec2_inventory.html) - - [Yandex Cloud (Note: Not tested)](https://github.com/rodion-goritskov/yacloud_compute) - - Implementing dynamic inventory can enhance your automation capabilities. - -2. Secure Docker Configuration: - - Add a task to configure Docker security settings, disable root access. - - Use the `copy` module and modify the `daemon.json` file. - -### Guidelines - -- Use proper Markdown formatting and structure for documentation files. -- Organize files within the lab folder with suitable naming conventions. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Ensure that your repository is well-structured, follow Ansible best practices, and provide clear documentation for a successful submission. diff --git a/lab6.md b/lab6.md deleted file mode 100644 index cc8249390d..0000000000 --- a/lab6.md +++ /dev/null @@ -1,139 +0,0 @@ -# Lab 6: Ansible and Application Deployment - -## Overview - -In this lab, you will utilize Ansible to set up a Continuous Deployment (CD) process for your application. - -## Task 1: Application Deployment - -**6 Points:** - -1. Create an Ansible Role: - - Develop an Ansible role specifically for deploying your application's Docker image, it can be done manually or via `ansible-galaxy init roles/web_app`. Call it `web_app`. - - Define variables in `roles/web_app/defaults/main.yml`. - - Add tasks to `roles/web_app/tasks/main.yml` to pull the Docker image and start the container. - - > Managing just a container is bad practice, you can omit it and move to the Task 2 directly. - -2. Update the Playbook: - - Modify your Ansible playbook to integrate the new role you've created for Docker image deployment. - -3. Deployment Output: - - Execute your playbook to deploy the role. - - Provide the last 50 lines of the output from your deployment command in the `ANSIBLE.md` file. - -## Task 2: Ansible Best Practices - -**4 Points:** - -1. Group Tasks with Blocks: - - Organize related tasks within your playbooks using Ansible blocks. - - Implement logical blocks. For example: - - ```yaml - - name: Setup Docker Environment - block: - - name: Install Docker - apt: - name: docker.io - state: present - - - name: Start Docker Service - service: - name: docker - state: started - enabled: yes - tags: - - setup - ``` - -2. Role Dependency: - - Set the role dependency for your `web_app` role to include the `docker` role. - - Specify dependencies in `roles/web_app/meta/main.yml`. - -3. Apply Tags: - - Implement Ansible tags to group tasks logically and enable selective execution. For example: - - ```yaml - - name: Pull Docker image - docker_image: - name: "{{ docker_image }}" - source: pull - tags: - - docker - ``` - - - Run specific tags. For example: - - ```bash - ansible-playbook site.yml --tags docker - ``` - -4. Wipe Logic: - - Create a wipe logic in `roles/web_app/tasks/0-wipe.yml`. This should include removing your Docker container and all related files. - - Ensure that this wipe process can be enabled or disabled by using a variable, for example, `web_app_full_wipe=true`. - -5. Separate Tag for Wipe: - - Utilize a distinct tag for the **Wipe** section of your Ansible playbook. This allows you to run the wipe tasks independently from the main tasks. - -6. Docker Compose File: - - Write a Jinja2 template (`roles/web_app/templates/docker-compose.yml.j2`). For example: - - ```yaml - version: '3' - services: - app: - image: "{{ docker_image }}" - ports: - - "{{ app_port }}:80" - ``` - - - Deliver the template using the `template` module in `roles/web_app/tasks/main.yml`. - - Suggested structure: - - ```sh - . - |-- defaults - | `-- main.yml - |-- meta - | `-- main.yml - |-- tasks - | |-- 0-wipe.yml - | `-- main.yml - `-- templates - `-- docker-compose.yml.j2 - ``` - -7. Create `README.md`: - - Create a `README.md` file in the `ansible/roles/web_app` folder. - - Use a suggested Docker Markdown template from the previous lab to describe your role, its requirements and usage. - -## Bonus Task: CD Improvement - -**2.5 Points:** - -1. Create an Extra Playbook: - - Develop an additional Ansible playbook specifically for your bonus application. - - You can reuse the existing Ansible role you created for your primary application or create a new one. - - Suggested structure: - - ```sh - . - `--ansible - `-- playbooks - `-- dev - |-- app_python - | `-- main.yaml - `-- app_go - `-- main.yaml - ``` - -### Guidelines - -- Use proper Markdown formatting and structure for documentation files. -- Organize files within the lab folder with suitable naming conventions. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. -- Follow the suggested structure for your Ansible roles, tasks, and templates. -- Utilize Ansible best practices such as grouping tasks with blocks, applying tags, and separating roles logically. - -> Note: Apply diligence to your Ansible implementation, follow best practices, and clearly document your work to achieve the best results in this lab assignment. diff --git a/lab7.md b/lab7.md deleted file mode 100644 index 48e65eb202..0000000000 --- a/lab7.md +++ /dev/null @@ -1,59 +0,0 @@ -# Lab 7: Monitoring and Logging - -## Overview - -In this lab, you will become familiar with a logging stack that includes Promtail, Loki, and Grafana. Your goal is to create a Docker Compose configuration and configuration files to set up this logging stack. - -## Task 1: Logging Stack Setup - -**6 Points:** - -1. Study the Logging Stack: - - Begin by researching the components of the logging stack: - - [Grafana Webinar: Loki Getting Started](https://grafana.com/go/webinar/loki-getting-started/) - - [Loki Overview](https://grafana.com/docs/loki/latest/overview/) - - [Loki GitHub Repository](https://github.com/grafana/loki) - -2. Create a Monitoring Folder: - - Start by creating a new folder named `monitoring` in your project directory. - -3. Docker Compose Configuration: - - Inside the `monitoring` folder, prepare a `docker-compose.yml` file that defines the entire logging stack along with your application. - - To assist you in this task, refer to these resources for sample Docker Compose configurations: - - [Example Docker Compose Configuration from Loki Repository](https://github.com/grafana/loki/blob/main/production/docker-compose.yaml) - - [Promtail Configuration Example](https://github.com/black-rosary/loki-nginx/blob/master/promtail/promtail.yml) (Adapt it as needed) - -4. Testing: - - Verify that the configured logging stack and your application work as expected. - -## Task 2: Documentation and Reporting - -**4 Points:** - -1. Logging Stack Report: - - Create a new file named `LOGGING.md` to document how the logging stack you've set up functions. - - Provide detailed explanations of each component's role within the stack. - -2. Screenshots: - - Capture screenshots that demonstrate the successful operation of your logging stack. - - Include these screenshots in your `LOGGING.md` report for reference. - -## Bonus Task: Additional Configuration - -**2.5 Points:** - -1. Integrating Your Extra App: - - Extend the `docker-compose.yml` configuration to include your additional application. - -2. Configure Stack for Comprehensive Logging: - - Modify the logging stack's configuration to collect logs from all containers defined in the `docker-compose.yml`. - - Include screenshots in your `LOGGING.md` report to demonstrate your success. - -### Guidelines - -- Ensure that your documentation in `LOGGING.md` is well-structured and comprehensible. -- Follow proper naming conventions for files and folders. -- Use code blocks and Markdown formatting where appropriate. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Thoroughly document your work, and ensure the logging stack functions correctly. Utilize the bonus points opportunity to enhance your understanding and the completeness of your setup. diff --git a/lab8.md b/lab8.md deleted file mode 100644 index 8eb0752ec7..0000000000 --- a/lab8.md +++ /dev/null @@ -1,71 +0,0 @@ -# Lab 8: Monitoring with Prometheus - -## Overview - -In this lab, you will become acquainted with Prometheus, set it up, and configure applications to collect metrics. - -## Task 1: Prometheus Setup - -**6 Points:** - -1. Learn About Prometheus: - - Begin by reading about Prometheus and its fundamental concepts: - - [Prometheus Overview](https://prometheus.io/docs/introduction/overview/) - - [Prometheus Naming Best Practices](https://prometheus.io/docs/practices/naming/) - -2. Integration with Docker Compose: - - Expand your existing `docker-compose.yml` file from the previous lab to include Prometheus. - -3. Prometheus Configuration: - - Configure Prometheus to collect metrics from both Loki and Prometheus containers. - -4. Verify Prometheus Targets: - - Access `http://localhost:9090/targets` to ensure that Prometheus is correctly scraping metrics. - - Capture screenshots that confirm the successful setup and place them in a file named `METRICS.md` within the monitoring folder. - -## Task 2: Dashboard and Configuration Enhancements - -**4 Points:** - -1. Grafana Dashboards: - - Set up dashboards in Grafana for both Loki and Prometheus. - - You can use examples as references: - - [Example Dashboard for Loki](https://grafana.com/grafana/dashboards/13407) - - [Example Dashboard for Prometheus](https://grafana.com/grafana/dashboards/3662) - - Capture screenshots displaying your successful dashboard configurations and include them in `METRICS.md`. - -2. Service Configuration Updates: - - Enhance the configuration of all services in the `docker-compose.yml` file: - - Add log rotation mechanisms. - - Specify memory limits for containers. - - Ensure these changes are documented within your `METRICS.md` file. - -3. Metrics Gathering: - - Extend Prometheus to gather metrics from all services defined in the `docker-compose.yml` file. - -## Bonus Task: Metrics and Health Checks - -**To Earn 2.5 Additional Points:** - -1. Application Metrics: - - Integrate metrics into your applications. You can refer to Python examples like: - - [Monitoring a Synchronous Python Web Application](https://dzone.com/articles/monitoring-your-synchronous-python-web-application) - - [Metrics Monitoring in Python](https://opensource.com/article/18/4/metrics-monitoring-and-python) - -2. Obtain Application Metrics: - - Configure your applications to export metrics. - -3. METRICS.md Update: - - Document your progress with the bonus tasks, including screenshots, in the `METRICS.md` file. - -4. Health Checks: - - Further enhance the `docker-compose.yml` file's service configurations by adding health checks for the containers. - -### Guidelines - -- Maintain a well-structured and comprehensible `METRICS.md` document. -- Adhere to file and folder naming conventions. -- Utilize code blocks and Markdown formatting where appropriate. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Ensure thorough documentation of your work, and guarantee that Prometheus correctly collects metrics. Take advantage of the bonus tasks to deepen your understanding and enhance the completeness of your setup. diff --git a/lab9.md b/lab9.md deleted file mode 100644 index 5493f042a6..0000000000 --- a/lab9.md +++ /dev/null @@ -1,76 +0,0 @@ -# Lab 9: Introduction to Kubernetes - -## Overview - -In this lab, you will explore Kubernetes, set up a local development environment, and create manifests for your application. - -## Task 1: Kubernetes Setup and Basic Deployment - -**6 Points:** - -1. Learn About Kubernetes: - - Begin by studying the fundamentals of Kubernetes: - - [What is Kubernetes](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/) - - [Kubernetes Components](https://kubernetes.io/docs/concepts/overview/components/) - -2. Install Kubernetes Tools: - - Install `kubectl` and `minikube`, essential tools for managing Kubernetes. - - [Kubernetes Tools](https://kubernetes.io/docs/tasks/tools/) - -3. Deploy Your Application: - - Deploy your application within the Minikube cluster using the `kubectl create` command. Create a `Deployment` resource for your app. - - [Example of Creating a Deployment](https://kubernetes.io/docs/tutorials/hello-minikube/#create-a-deployment) - - [Deployment Overview](https://kubernetes.io/docs/tutorials/kubernetes-basics/deploy-app/deploy-intro/) - -4. Access Your Application: - - Make your application accessible from outside the Kubernetes virtual network. Achieve this by creating a `Service` resource. - - [Example of Creating a Service](https://kubernetes.io/docs/tutorials/hello-minikube/#create-a-service) - - [Service Overview](https://kubernetes.io/docs/tutorials/kubernetes-basics/expose/expose-intro/) - -5. Create a Kubernetes Folder: - - Establish a `k8s` folder within your repository. - - Create a `README.md` report within this folder and include the output of the `kubectl get pods,svc` command. - -6. Cleanup: - - Remove the `Deployment` and `Service` resources that you created, maintaining a tidy Kubernetes environment. - -## Task 2: Declarative Kubernetes Manifests - -**4 Points:** - -1. Manifest Files for Your Application: - - As a more efficient and structured approach, employ configuration files to deploy your application. - - Create a `deployment.yml` manifest file that describes your app's deployment, specifying at least 3 replicas. - - [Kubernetes Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) - - [Declarative Management of Kubernetes Objects Using Configuration Files](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/) - -2. Service Manifest: - - Develop a `service.yml` manifest file for your application. - -3. Manifest Files in `k8s` Folder: - - Store these manifest files in the `k8s` folder of your repository. - - Additionally, provide the output of the `kubectl get pods,svc` command in the `README.md` report. - - Include the output of the `minikube service --all` command and the result from your browser, with a screenshot demonstrating that the IP matches the output of `minikube service --all`. - -## Bonus Task: Additional Configuration and Ingress - -**To Earn 2.5 Additional Points:** - -1. Manifests for Extra App: - - Create `deployment` and `service` manifests for an additional application. - -2. Ingress Manifests: - - Construct [Ingress manifests](https://kubernetes.io/docs/tasks/access-application-cluster/ingress-minikube/) for your applications. - -3. Application Availability Check: - - Utilize `curl` or a similar tool to verify the availability of your applications. Include the output in the report. - -**Guidelines:** - -- Maintain a clear and well-structured `README.md` document. -- Ensure that all required components are included. -- Adhere to file and folder naming conventions. -- Create and participate in PRs to facilitate the peer review process. -- Create pull requests (PRs) as needed: from your fork to the main branch of this repository, and from your fork's branch to your fork's master branch. - -> Note: Detailed documentation is crucial to ensure that your Kubernetes deployment is fully functional and accessible. Engage with the bonus tasks to further enhance your understanding and application deployment skills. diff --git a/labs/lab01.md b/labs/lab01.md new file mode 100644 index 0000000000..18c9ff6c43 --- /dev/null +++ b/labs/lab01.md @@ -0,0 +1,693 @@ +# Lab 1 — DevOps Info Service: Web Application Development + +![difficulty](https://img.shields.io/badge/difficulty-beginner-success) +![topic](https://img.shields.io/badge/topic-Web%20Development-blue) +![points](https://img.shields.io/badge/points-10%2B2.5-orange) +![languages](https://img.shields.io/badge/languages-Python%20|%20Go-informational) + +> Build a DevOps info service that reports system information and health status. This service will evolve throughout the course into a comprehensive monitoring tool. + +## Overview + +Create a **DevOps Info Service** - a web application providing detailed information about itself and its runtime environment. This foundation will grow throughout the course as you add containerization, CI/CD, monitoring, and persistence. + +**What You'll Learn:** +- Web framework selection and implementation +- System introspection and API design +- Python best practices and documentation +- Foundation for future DevOps tooling + +**Tech Stack:** Python 3.11+ | Flask 3.1 or FastAPI 0.115 + +--- + +## Tasks + +### Task 1 — Python Web Application (6 pts) + +Build a production-ready Python web service with comprehensive system information. + +#### 1.1 Project Structure + +Create this structure: + +``` +app_python/ +├── app.py # Main application +├── requirements.txt # Dependencies +├── .gitignore # Git ignore +├── README.md # App documentation +├── tests/ # Unit tests (Lab 3) +│ └── __init__.py +└── docs/ # Lab documentation + ├── LAB01.md # Your lab submission + └── screenshots/ # Proof of work + ├── 01-main-endpoint.png + ├── 02-health-check.png + └── 03-formatted-output.png +``` + +#### 1.2 Choose Web Framework + +Select and justify your choice: +- **Flask** - Lightweight, easy to learn +- **FastAPI** - Modern, async, auto-documentation +- **Django** - Full-featured, includes ORM + +Document your decision in `app_python/docs/LAB01.md`. + +#### 1.3 Implement Main Endpoint: `GET /` + +Return comprehensive service and system information: + +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Flask" + }, + "system": { + "hostname": "my-laptop", + "platform": "Linux", + "platform_version": "Ubuntu 24.04", + "architecture": "x86_64", + "cpu_count": 8, + "python_version": "3.13.1" + }, + "runtime": { + "uptime_seconds": 3600, + "uptime_human": "1 hour, 0 minutes", + "current_time": "2026-01-07T14:30:00.000Z", + "timezone": "UTC" + }, + "request": { + "client_ip": "127.0.0.1", + "user_agent": "curl/7.81.0", + "method": "GET", + "path": "/" + }, + "endpoints": [ + {"path": "/", "method": "GET", "description": "Service information"}, + {"path": "/health", "method": "GET", "description": "Health check"} + ] +} +``` + +
+💡 Implementation Hints + +**Get System Information:** +```python +import platform +import socket +from datetime import datetime + +hostname = socket.gethostname() +platform_name = platform.system() +architecture = platform.machine() +python_version = platform.python_version() +``` + +**Calculate Uptime:** +```python +start_time = datetime.now() + +def get_uptime(): + delta = datetime.now() - start_time + seconds = int(delta.total_seconds()) + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + return { + 'seconds': seconds, + 'human': f"{hours} hours, {minutes} minutes" + } +``` + +**Request Information:** +```python +# Flask +request.remote_addr # Client IP +request.headers.get('User-Agent') # User agent +request.method # HTTP method +request.path # Request path + +# FastAPI +request.client.host +request.headers.get('user-agent') +request.method +request.url.path +``` + +
+ +#### 1.4 Implement Health Check: `GET /health` + +Simple health endpoint for monitoring: + +```json +{ + "status": "healthy", + "timestamp": "2024-01-15T14:30:00.000Z", + "uptime_seconds": 3600 +} +``` + +Return HTTP 200 for healthy status. This will be used for Kubernetes probes in Lab 9. + +
+💡 Implementation Hints + +```python +# Flask +@app.route('/health') +def health(): + return jsonify({ + 'status': 'healthy', + 'timestamp': datetime.now(timezone.utc).isoformat(), + 'uptime_seconds': get_uptime()['seconds'] + }) + +# FastAPI +@app.get("/health") +def health(): + return { + 'status': 'healthy', + 'timestamp': datetime.now(timezone.utc).isoformat(), + 'uptime_seconds': get_uptime()['seconds'] + } +``` + +
+ +#### 1.5 Configuration + +Make your app configurable via environment variables: + +```python +import os + +HOST = os.getenv('HOST', '0.0.0.0') +PORT = int(os.getenv('PORT', 5000)) +DEBUG = os.getenv('DEBUG', 'False').lower() == 'true' +``` + +**Test:** +```bash +python app.py # Default: 0.0.0.0:5000 +PORT=8080 python app.py # Custom port +HOST=127.0.0.1 PORT=3000 python app.py +``` + +--- + +### Task 2 — Documentation & Best Practices (4 pts) + +#### 2.1 Application README (`app_python/README.md`) + +Create user-facing documentation: + +**Required Sections:** +1. **Overview** - What the service does +2. **Prerequisites** - Python version, dependencies +3. **Installation** + ```bash + python -m venv venv + source venv/bin/activate + pip install -r requirements.txt + ``` +4. **Running the Application** + ```bash + python app.py + # Or with custom config + PORT=8080 python app.py + ``` +5. **API Endpoints** + - `GET /` - Service and system information + - `GET /health` - Health check +6. **Configuration** - Environment variables table + +#### 2.2 Best Practices + +Implement these in your code: + +**1. Clean Code Organization** +- Clear function names +- Proper imports grouping +- Comments only where needed +- Follow PEP 8 + +
+💡 Example Structure + +```python +""" +DevOps Info Service +Main application module +""" +import os +import socket +import platform +from datetime import datetime, timezone +from flask import Flask, jsonify, request + +app = Flask(__name__) + +# Configuration +HOST = os.getenv('HOST', '0.0.0.0') +PORT = int(os.getenv('PORT', 5000)) + +# Application start time +START_TIME = datetime.now(timezone.utc) + +def get_system_info(): + """Collect system information.""" + return { + 'hostname': socket.gethostname(), + 'platform': platform.system(), + 'architecture': platform.machine(), + 'python_version': platform.python_version() + } + +@app.route('/') +def index(): + """Main endpoint - service and system information.""" + # Implementation +``` + +
+ +**2. Error Handling** + +
+💡 Implementation + +```python +@app.errorhandler(404) +def not_found(error): + return jsonify({ + 'error': 'Not Found', + 'message': 'Endpoint does not exist' + }), 404 + +@app.errorhandler(500) +def internal_error(error): + return jsonify({ + 'error': 'Internal Server Error', + 'message': 'An unexpected error occurred' + }), 500 +``` + +
+ +**3. Logging** + +
+💡 Implementation + +```python +import logging + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +logger.info('Application starting...') +logger.debug(f'Request: {request.method} {request.path}') +``` + +
+ +**4. Dependencies (`requirements.txt`)** + +```txt +# Web Framework +Flask==3.1.0 +# or +fastapi==0.115.0 +uvicorn[standard]==0.32.0 # Includes performance extras +``` + +Pin exact versions for reproducibility. + +**5. Git Ignore (`.gitignore`)** + +```gitignore +# Python +__pycache__/ +*.py[cod] +venv/ +*.log + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store +``` + +#### 2.3 Lab Submission (`app_python/docs/LAB01.md`) + +Document your implementation: + +**Required Sections:** +1. **Framework Selection** + - Your choice and why + - Comparison table with alternatives +2. **Best Practices Applied** + - List practices with code examples + - Explain importance of each +3. **API Documentation** + - Request/response examples + - Testing commands +4. **Testing Evidence** + - Screenshots showing endpoints work + - Terminal output +5. **Challenges & Solutions** + - Problems encountered + - How you solved them + +**Required Screenshots:** +- Main endpoint showing complete JSON +- Health check response +- Formatted/pretty-printed output + +#### 2.4 GitHub Community Engagement + +**Objective:** Explore GitHub's social features that support collaboration and discovery. + +**Actions Required:** +1. **Star** the course repository +2. **Star** the [simple-container-com/api](https://github.com/simple-container-com/api) project — a promising open-source tool for container management +3. **Follow** your professor and TAs on GitHub: + - Professor: [@Cre-eD](https://github.com/Cre-eD) + - TA: [@marat-biriushev](https://github.com/marat-biriushev) + - TA: [@pierrepicaud](https://github.com/pierrepicaud) +4. **Follow** at least 3 classmates from the course + +**Document in LAB01.md:** + +Add a "GitHub Community" section (after Challenges & Solutions) with 1-2 sentences explaining: +- Why starring repositories matters in open source +- How following developers helps in team projects and professional growth + +
+💡 GitHub Social Features + +**Why Stars Matter:** + +**Discovery & Bookmarking:** +- Stars help you bookmark interesting projects for later reference +- Star count indicates project popularity and community trust +- Starred repos appear in your GitHub profile, showing your interests + +**Open Source Signal:** +- Stars encourage maintainers (shows appreciation) +- High star count attracts more contributors +- Helps projects gain visibility in GitHub search and recommendations + +**Professional Context:** +- Shows you follow best practices and quality projects +- Indicates awareness of industry tools and trends + +**Why Following Matters:** + +**Networking:** +- See what other developers are working on +- Discover new projects through their activity +- Build professional connections beyond the classroom + +**Learning:** +- Learn from others' code and commits +- See how experienced developers solve problems +- Get inspiration for your own projects + +**Collaboration:** +- Stay updated on classmates' work +- Easier to find team members for future projects +- Build a supportive learning community + +**Career Growth:** +- Follow thought leaders in your technology stack +- See trending projects in real-time +- Build visibility in the developer community + +**GitHub Best Practices:** +- Star repos you find useful (not spam) +- Follow developers whose work interests you +- Engage meaningfully with the community +- Your GitHub activity shows employers your interests and involvement + +
+ +--- + +## Bonus Task — Compiled Language (2.5 pts) + +Implement the same service in a compiled language to prepare for multi-stage Docker builds (Lab 2). + +**Choose One:** +- **Go** (Recommended) - Small binaries, fast compilation +- **Rust** - Memory safety, modern features +- **Java/Spring Boot** - Enterprise standard +- **C#/ASP.NET Core** - Cross-platform .NET + +**Structure:** + +``` +app_go/ (or app_rust, app_java, etc.) +├── main.go +├── go.mod +├── README.md +└── docs/ + ├── LAB01.md # Implementation details + ├── GO.md # Language justification + └── screenshots/ +``` + +**Requirements:** +- Same two endpoints: `/` and `/health` +- Same JSON structure +- Document build process +- Compare binary size to Python + +
+💡 Go Example Skeleton + +```go +package main + +import ( + "encoding/json" + "net/http" + "os" + "runtime" + "time" +) + +type ServiceInfo struct { + Service Service `json:"service"` + System System `json:"system"` + Runtime Runtime `json:"runtime"` + Request Request `json:"request"` +} + +var startTime = time.Now() + +func mainHandler(w http.ResponseWriter, r *http.Request) { + info := ServiceInfo{ + Service: Service{ + Name: "devops-info-service", + Version: "1.0.0", + }, + System: System{ + Platform: runtime.GOOS, + Architecture: runtime.GOARCH, + CPUCount: runtime.NumCPU(), + }, + // ... implement rest + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(info) +} + +func main() { + http.HandleFunc("/", mainHandler) + http.HandleFunc("/health", healthHandler) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + + http.ListenAndServe(":"+port, nil) +} +``` + +
+ +--- + +## How to Submit + +1. **Create Branch:** + ```bash + git checkout -b lab01 + ``` + +2. **Commit Work:** + ```bash + git add app_python/ + git commit -m "feat: implement lab01 devops info service" + git push -u origin lab01 + ``` + +3. **Create Pull Requests:** + - **PR #1:** `your-fork:lab01` → `course-repo:master` + - **PR #2:** `your-fork:lab01` → `your-fork:master` + +4. **Verify:** + - All files present + - Screenshots included + - Documentation complete + +--- + +## Acceptance Criteria + +### Main Tasks (10 points) + +**Application Functionality (3 pts):** +- [ ] Service runs without errors +- [ ] `GET /` returns all required fields: + - [ ] Service metadata (name, version, description, framework) + - [ ] System info (hostname, platform, architecture, CPU, Python version) + - [ ] Runtime info (uptime, current time, timezone) + - [ ] Request info (client IP, user agent, method, path) + - [ ] Endpoints list +- [ ] `GET /health` returns status and uptime +- [ ] Configurable via environment variables (PORT, HOST) + +**Code Quality (2 pts):** +- [ ] Clean code structure +- [ ] PEP 8 compliant +- [ ] Error handling implemented +- [ ] Logging configured + +**Documentation (3 pts):** +- [ ] `app_python/README.md` complete with all sections +- [ ] `app_python/docs/LAB01.md` includes: + - [ ] Framework justification + - [ ] Best practices documentation + - [ ] API examples + - [ ] Testing evidence + - [ ] Challenges solved + - [ ] GitHub Community section (why stars/follows matter) +- [ ] All 3 required screenshots present +- [ ] Course repository starred +- [ ] simple-container-com/api repository starred +- [ ] Professor and TAs followed on GitHub +- [ ] At least 3 classmates followed on GitHub + +**Configuration (2 pts):** +- [ ] `requirements.txt` with pinned versions +- [ ] `.gitignore` properly configured +- [ ] Environment variables working + +### Bonus Task (2.5 points) + +- [ ] Compiled language app implements both endpoints +- [ ] Same JSON structure as Python version +- [ ] `app_/README.md` with build/run instructions +- [ ] `app_/docs/GO.md` with language justification +- [ ] `app_/docs/LAB01.md` with implementation details +- [ ] Screenshots showing compilation and execution + +--- + +## Rubric + +| Criteria | Points | Description | +|----------|--------|-------------| +| **Functionality** | 3 pts | Both endpoints work with complete, correct data | +| **Code Quality** | 2 pts | Clean, organized, follows Python standards | +| **Documentation** | 3 pts | Complete README and lab submission docs | +| **Configuration** | 2 pts | Dependencies, environment vars, .gitignore | +| **Bonus** | 2.5 pts | Compiled language implementation | +| **Total** | 12.5 pts | 10 pts required + 2.5 pts bonus | + +**Grading Scale:** +- **10/10:** Perfect implementation, excellent documentation +- **8-9/10:** All works, good docs, minor improvements possible +- **6-7/10:** Core functionality present, basic documentation +- **<6/10:** Missing features or documentation, needs revision + +--- + +## Resources + +
+📚 Python Web Frameworks + +- [Flask 3.1 Documentation](https://flask.palletsprojects.com/en/latest/) +- [Flask Quickstart](https://flask.palletsprojects.com/en/latest/quickstart/) +- [FastAPI Documentation](https://fastapi.tiangolo.com/) +- [FastAPI Tutorial](https://fastapi.tiangolo.com/tutorial/first-steps/) +- [Django 5.1 Documentation](https://docs.djangoproject.com/en/5.1/) + +
+ +
+🐍 Python Best Practices + +- [PEP 8 Style Guide](https://pep8.org/) +- [Python Logging Tutorial](https://docs.python.org/3/howto/logging.html) +- [Python platform module](https://docs.python.org/3/library/platform.html) +- [Python socket module](https://docs.python.org/3/library/socket.html) + +
+ +
+🔧 Compiled Languages (Bonus) + +- [Go Web Development](https://golang.org/doc/articles/wiki/) +- [Go net/http Package](https://pkg.go.dev/net/http) +- [Rust Web Frameworks](https://www.arewewebyet.org/) +- [Spring Boot Quickstart](https://spring.io/quickstart) +- [ASP.NET Core Tutorial](https://docs.microsoft.com/aspnet/core/) + +
+ +
+🛠️ Development Tools + +- [Postman](https://www.postman.com/) - API testing +- [HTTPie](https://httpie.io/) - Command-line HTTP client +- [curl](https://curl.se/) - Data transfer tool +- [jq](https://stedolan.github.io/jq/) - JSON processor + +
+ +--- + +## Looking Ahead + +This service evolves throughout the course: + +- **Lab 2:** Containerize with Docker, multi-stage builds +- **Lab 3:** Add unit tests and CI/CD pipeline +- **Lab 8:** Add `/metrics` endpoint for Prometheus +- **Lab 9:** Deploy to Kubernetes using `/health` probes +- **Lab 12:** Add `/visits` endpoint with file persistence +- **Lab 13:** Multi-environment deployment with GitOps + +--- + +**Good luck!** 🚀 + +> **Remember:** Keep it simple, write clean code, and document thoroughly. This foundation will carry through all 16 labs! diff --git a/labs/lab02.md b/labs/lab02.md new file mode 100644 index 0000000000..1c3e032f89 --- /dev/null +++ b/labs/lab02.md @@ -0,0 +1,366 @@ +# Lab 2 — Docker Containerization + +![difficulty](https://img.shields.io/badge/difficulty-beginner-success) +![topic](https://img.shields.io/badge/topic-Containerization-blue) +![points](https://img.shields.io/badge/points-10%2B2.5-orange) +![tech](https://img.shields.io/badge/tech-Docker-informational) + +> Containerize your Python app from Lab 1 using Docker best practices and publish it to Docker Hub. + +## Overview + +Take your Lab 1 application and package it into a Docker container. Learn image optimization, security basics, and the Docker workflow used in production. + +**What You'll Learn:** +- Writing production-ready Dockerfiles +- Docker best practices and security +- Image optimization techniques +- Docker Hub workflow + +**Tech Stack:** Docker 25+ | Python 3.13-slim | Multi-stage builds + +--- + +## Tasks + +### Task 1 — Create Dockerfile (4 pts) + +**Objective:** Write a Dockerfile that containerizes your Python app following best practices. + +Create `app_python/Dockerfile` with these requirements: + +**Must Have:** +- Non-root user (mandatory) +- Specific base image version (e.g., `python:3.13-slim` or `python:3.12-slim`) +- Only copy necessary files +- Proper layer ordering +- `.dockerignore` file + +**Your app should work the same way in the container as it did locally.** + +
+💡 Dockerfile Concepts & Resources + +**Key Dockerfile Instructions to Research:** +- `FROM` - Choose your base image (look at python:3.13-slim, python:3.12-slim, python:3.13-alpine) +- `RUN` - Execute commands (creating users, installing packages) +- `WORKDIR` - Set working directory +- `COPY` - Copy files into the image +- `USER` - Switch to non-root user +- `EXPOSE` - Document which port your app uses +- `CMD` - Define how to start your application + +**Critical Concepts:** +- **Layer Caching**: Why does the order of COPY commands matter? +- **Non-root User**: How do you create and switch to a non-root user? +- **Base Image Selection**: What's the difference between slim, alpine, and full images? +- **Dependency Installation**: Why copy requirements.txt separately from application code? + +**Resources:** +- [Dockerfile Reference](https://docs.docker.com/reference/dockerfile/) +- [Best Practices Guide](https://docs.docker.com/build/building/best-practices/) +- [Python Image Variants](https://hub.docker.com/_/python) - Use 3.13-slim or 3.12-slim + +**Think About:** +- What happens if you copy all files before installing dependencies? +- Why shouldn't you run as root? +- How does layer caching speed up rebuilds? + +
+ +
+💡 .dockerignore Concepts + +**Purpose:** Prevent unnecessary files from being sent to Docker daemon during build (faster builds, smaller context). + +**What Should You Exclude?** +Think about what doesn't need to be in your container: +- Development artifacts (like Python's `__pycache__`, `*.pyc`) +- Version control files (`.git` directory) +- IDE configuration files +- Virtual environments (`venv/`, `.venv/`) +- Documentation that's not needed at runtime +- Test files (if not running tests in container) + +**Key Question:** Why does excluding files from the build context matter for build speed? + +**Resources:** +- [.dockerignore Documentation](https://docs.docker.com/engine/reference/builder/#dockerignore-file) +- Look at your `.gitignore` for inspiration - many patterns overlap + +**Exercise:** Start minimal and add exclusions as needed, rather than copying a huge list you don't understand. + +
+ +**Test Your Container:** + +You should be able to: +1. Build your image using the `docker build` command +2. Run a container from your image with proper port mapping +3. Access your application endpoints from the host machine + +Verify that your application works the same way in the container as it did locally. + +--- + +### Task 2 — Docker Hub (2 pts) + +**Objective:** Publish your image to Docker Hub. + +**Requirements:** +1. Create a Docker Hub account (if you don't have one) +2. Tag your image with your Docker Hub username +3. Authenticate with Docker Hub +4. Push your image to the registry +5. Verify the image is publicly accessible + +**Documentation Required:** +- Terminal output showing successful push +- Docker Hub repository URL +- Explanation of your tagging strategy + +
+💡 Docker Hub Resources + +**Useful Commands:** +- `docker tag` - Tag images for registry push +- `docker login` - Authenticate with Docker Hub +- `docker push` - Upload image to registry +- `docker pull` - Download image from registry + +**Resources:** +- [Docker Hub Quickstart](https://docs.docker.com/docker-hub/quickstart/) +- [Docker Tag Reference](https://docs.docker.com/reference/cli/docker/image/tag/) +- [Best Practices for Tagging](https://docs.docker.com/build/building/best-practices/#tagging) + +
+ +--- + +### Task 3 — Documentation (4 pts) + +**Objective:** Document your Docker implementation with focus on understanding and decisions. + +#### 3.1 Update `app_python/README.md` + +Add a **Docker** section explaining how to use your containerized application. Include command patterns (not exact commands) for: +- Building the image locally +- Running a container +- Pulling from Docker Hub + +#### 3.2 Create `app_python/docs/LAB02.md` + +Document your implementation with these sections: + +**Required Sections:** + +1. **Docker Best Practices Applied** + - List each practice you implemented (non-root user, layer caching, .dockerignore, etc.) + - Explain WHY each matters (not just what it does) + - Include relevant Dockerfile snippets with explanations + +2. **Image Information & Decisions** + - Base image chosen and justification (why this specific version?) + - Final image size and your assessment + - Layer structure explanation + - Optimization choices you made + +3. **Build & Run Process** + - Complete terminal output from your build process + - Terminal output showing container running + - Terminal output from testing endpoints (curl/httpie) + - Docker Hub repository URL + +4. **Technical Analysis** + - Why does your Dockerfile work the way it does? + - What would happen if you changed the layer order? + - What security considerations did you implement? + - How does .dockerignore improve your build? + +5. **Challenges & Solutions** + - Issues encountered during implementation + - How you debugged and resolved them + - What you learned from the process + +--- + +## Bonus Task — Multi-Stage Build (2.5 pts) + +**Objective:** Containerize your compiled language app (from Lab 1 bonus) using multi-stage builds. + +**Why Multi-Stage?** Separate build environment from runtime → smaller final image. + +**Example Flow:** +1. **Stage 1 (Builder):** Compile the app (large image with compilers) +2. **Stage 2 (Runtime):** Copy only the binary (small image, no build tools) + +
+💡 Multi-Stage Build Concepts + +**The Problem:** Compiled language images include the entire compiler/SDK in the final image (huge!). + +**The Solution:** Use multiple `FROM` statements: +- **Stage 1 (Builder)**: Use full SDK image, compile your application +- **Stage 2 (Runtime)**: Use minimal base image, copy only the compiled binary + +**Key Concepts to Research:** +- How to name build stages (`AS builder`) +- How to copy files from previous stages (`COPY --from=builder`) +- Choosing runtime base images (alpine, distroless, scratch) +- Static vs dynamic compilation (affects what base image you can use) + +**Questions to Explore:** +- What's the size difference between your builder and final image? +- Why can't you just use the builder image as your final image? +- What security benefits come from smaller images? +- Can you use `FROM scratch`? Why or why not? + +**Resources:** +- [Multi-Stage Builds Documentation](https://docs.docker.com/build/building/multi-stage/) +- [Distroless Base Images](https://github.com/GoogleContainerTools/distroless) +- Language-specific: Search "Go static binary Docker" or "Rust alpine Docker" + +**Challenge:** Try to get your final image under 20MB. + +
+ +**Requirements:** +- Multi-stage Dockerfile in `app_go/` (or your chosen language) +- Working containerized application +- Documentation in `app_go/docs/LAB02.md` explaining: + - Your multi-stage build strategy + - Size comparison with analysis (builder vs final image) + - Why multi-stage builds matter for compiled languages + - Terminal output showing build process and image sizes + - Technical explanation of each stage's purpose + +**Bonus Points Given For:** +- Significant size reduction achieved with clear metrics +- Deep understanding of multi-stage build benefits +- Analysis of security implications (smaller attack surface) +- Explanation of trade-offs and decisions made + +--- + +## How to Submit + +1. **Create Branch:** Create a new branch called `lab02` + +2. **Commit Work:** + - Add your changes (app_python/ directory with Dockerfile, .dockerignore, updated docs) + - Commit with a descriptive message following conventional commits format + - Push to your fork + +3. **Create Pull Requests:** + - **PR #1:** `your-fork:lab02` → `course-repo:master` + - **PR #2:** `your-fork:lab02` → `your-fork:master` + +--- + +## Acceptance Criteria + +### Main Tasks (10 points) + +**Dockerfile (4 pts):** +- [ ] Dockerfile exists in `app_python/` +- [ ] Uses specific base image version +- [ ] Runs as non-root user (USER directive) +- [ ] Proper layer ordering (dependencies before code) +- [ ] Only copies necessary files +- [ ] `.dockerignore` file present +- [ ] Image builds successfully +- [ ] Container runs and app works + +**Docker Hub (2 pts):** +- [ ] Image pushed to Docker Hub +- [ ] Image is publicly accessible +- [ ] Correct tagging used +- [ ] Can pull and run from Docker Hub + +**Documentation (4 pts):** +- [ ] `app_python/README.md` has Docker section with command patterns +- [ ] `app_python/docs/LAB02.md` complete with: + - [ ] Best practices explained with WHY (not just what) + - [ ] Image information and justifications for choices + - [ ] Terminal output from build, run, and testing + - [ ] Technical analysis demonstrating understanding + - [ ] Challenges and solutions documented + - [ ] Docker Hub repository URL provided + +### Bonus Task (2.5 points) + +- [ ] Multi-stage Dockerfile for compiled language app +- [ ] Working containerized application +- [ ] Documentation in `app_/docs/LAB02.md` with: + - [ ] Multi-stage strategy explained + - [ ] Terminal output showing image sizes (builder vs final) + - [ ] Analysis of size reduction and why it matters + - [ ] Technical explanation of each stage + - [ ] Security benefits discussed + +--- + +## Rubric + +| Criteria | Points | Description | +|----------|--------|-------------| +| **Dockerfile** | 4 pts | Correct, secure, optimized | +| **Docker Hub** | 2 pts | Successfully published | +| **Documentation** | 4 pts | Complete and clear | +| **Bonus** | 2.5 pts | Multi-stage implementation | +| **Total** | 12.5 pts | 10 pts required + 2.5 pts bonus | + +**Grading:** +- **10/10:** Perfect Dockerfile, deep understanding demonstrated, excellent analysis +- **8-9/10:** Working container, good practices, solid understanding shown +- **6-7/10:** Container works, basic security, surface-level explanations +- **<6/10:** Missing requirements, runs as root, copy-paste without understanding + +--- + +## Resources + +
+📚 Docker Documentation + +- [Dockerfile Best Practices](https://docs.docker.com/build/building/best-practices/) +- [Dockerfile Reference](https://docs.docker.com/reference/dockerfile/) +- [Multi-Stage Builds](https://docs.docker.com/build/building/multi-stage/) +- [.dockerignore](https://docs.docker.com/reference/dockerfile/#dockerignore-file) +- [Docker Build Guide](https://docs.docker.com/build/guide/) + +
+ +
+🔒 Security Resources + +- [Docker Security Best Practices](https://docs.docker.com/build/building/best-practices/#security) +- [Snyk Docker Security](https://snyk.io/learn/docker-security-scanning/) +- [Why Non-Root Containers](https://docs.docker.com/build/building/best-practices/#user) +- [Distroless Images](https://github.com/GoogleContainerTools/distroless) - Minimal base images + +
+ +
+🛠️ Tools + +- [Hadolint](https://github.com/hadolint/hadolint) - Dockerfile linter +- [Dive](https://github.com/wagoodman/dive) - Explore image layers +- [Docker Hub](https://hub.docker.com/) - Container registry + +
+ +--- + +## Looking Ahead + +- **Lab 3:** CI/CD will automatically build these Docker images +- **Lab 7-8:** Deploy containers with docker-compose for logging/monitoring +- **Lab 9:** Run these containers in Kubernetes +- **Lab 13:** ArgoCD will deploy containerized apps automatically + +--- + +**Good luck!** 🚀 + +> **Remember:** Understanding beats copy-paste. Explain your decisions, not just your actions. Run as non-root or no points! diff --git a/labs/lab03.md b/labs/lab03.md new file mode 100644 index 0000000000..9824e934b3 --- /dev/null +++ b/labs/lab03.md @@ -0,0 +1,931 @@ +# Lab 3 — Continuous Integration (CI/CD) + +![difficulty](https://img.shields.io/badge/difficulty-beginner-success) +![topic](https://img.shields.io/badge/topic-CI/CD-blue) +![points](https://img.shields.io/badge/points-10%2B2.5-orange) +![tech](https://img.shields.io/badge/tech-GitHub%20Actions-informational) + +> Automate your Python app testing and Docker builds with GitHub Actions CI/CD pipeline. + +## Overview + +Take your containerized app from Labs 1-2 and add automated testing and deployment. Learn how CI/CD catches bugs early, ensures code quality, and automates the Docker build/push workflow. + +**What You'll Learn:** +- Writing effective unit tests +- GitHub Actions workflow syntax +- CI/CD best practices (caching, matrix builds, security scanning) +- Automated Docker image publishing +- Continuous integration for multiple applications + +**Tech Stack:** GitHub Actions | pytest 8+ | Python 3.11+ | Snyk | Docker + +**Connection to Previous Labs:** +- **Lab 1:** Test the endpoints you created +- **Lab 2:** Automate the Docker build/push workflow +- **Lab 4+:** This CI pipeline will run for all future labs + +--- + +## Tasks + +### Task 1 — Unit Testing (3 pts) + +**Objective:** Write comprehensive unit tests for your Python application to ensure reliability. + +**Requirements:** + +1. **Choose a Testing Framework** + - Research Python testing frameworks (pytest, unittest, etc.) + - Select one and justify your choice + - Install it in your `requirements.txt` or create `requirements-dev.txt` + +2. **Write Unit Tests** + - Create `app_python/tests/` directory + - Write tests for **all** your endpoints: + - `GET /` - Verify JSON structure and required fields + - `GET /health` - Verify health check response + - Test both successful responses and error cases + - Aim for meaningful test coverage (not just basic smoke tests) + +3. **Run Tests Locally** + - Verify all tests pass locally before CI setup + - Document how to run tests in your README + +
+💡 Testing Framework Guidance + +**Popular Python Testing Frameworks:** + +**pytest (Recommended):** +- Pros: Simple syntax, powerful fixtures, excellent plugin ecosystem +- Cons: Additional dependency +- Use case: Most modern Python projects + +**unittest:** +- Pros: Built into Python (no extra dependencies) +- Cons: More verbose, less modern features +- Use case: Minimal dependency projects + +**Key Testing Concepts to Research:** +- Test fixtures and setup/teardown +- Mocking external dependencies +- Testing HTTP endpoints (test client usage) +- Test coverage measurement +- Assertions and expected vs actual results + +**What Should You Test?** +- Correct HTTP status codes (200, 404, 500) +- Response data structure (JSON fields present) +- Response data types (strings, integers, etc.) +- Edge cases (invalid requests, missing data) +- Error handling (what happens when things fail?) + +**Questions to Consider:** +- How do you test a Flask/FastAPI app without starting the server? +- Should you test that `hostname` returns your actual hostname, or just that the field exists? +- How do you simulate different client IPs or user agents in tests? + +**Resources:** +- [Pytest Documentation](https://docs.pytest.org/) +- [Flask Testing](https://flask.palletsprojects.com/en/stable/testing/) +- [FastAPI Testing](https://fastapi.tiangolo.com/tutorial/testing/) +- [Python unittest](https://docs.python.org/3/library/unittest.html) + +**Anti-Patterns to Avoid:** +- Testing framework functionality instead of your code +- Tests that always pass regardless of implementation +- Tests with no assertions +- Tests that depend on external services + +
+ +**What to Document:** +- Your testing framework choice and why +- Test structure explanation +- How to run tests locally +- Terminal output showing all tests passing + +--- + +### Task 2 — GitHub Actions CI Workflow (4 pts) + +**Objective:** Create a GitHub Actions workflow that automatically tests your code and builds Docker images with proper versioning. + +**Requirements:** + +1. **Create Workflow File** + - Create `.github/workflows/python-ci.yml` in your repository + - Name your workflow descriptively + +2. **Implement Essential CI Steps** + + Your workflow must include these logical stages: + + **a) Code Quality & Testing:** + - Install dependencies + - Run a linter (pylint, flake8, black, ruff, etc.) + - Run your unit tests + + **b) Docker Build & Push with Versioning:** + - Authenticate with Docker Hub + - Build your Docker image + - Tag with proper version strategy (see versioning section below) + - Push to Docker Hub with multiple tags + +3. **Versioning Strategy** + + Choose **one** versioning approach and implement it: + + **Option A: Semantic Versioning (SemVer)** + - Version format: `v1.2.3` (major.minor.patch) + - Use git tags for releases + - Tag images like: `username/app:1.2.3`, `username/app:1.2`, `username/app:latest` + - **When to use:** Traditional software releases with breaking changes + + **Option B: Calendar Versioning (CalVer)** + - Version format: `2024.01.15` or `2024.01` (year.month.day or year.month) + - Based on release date + - Tag images like: `username/app:2024.01`, `username/app:latest` + - **When to use:** Time-based releases, continuous deployment + + **Required:** + - Document which strategy you chose and why + - Implement it in your CI workflow + - Show at least 2 tags per image (e.g., version + latest) + +4. **Workflow Triggers** + - Configure when the workflow runs (push, pull request, etc.) + - Consider which branches should trigger builds + +5. **Testing the Workflow** + - Push your workflow file and verify it runs + - Fix any issues that arise + - Ensure all steps complete successfully + - Verify Docker Hub shows your version tags + +
+💡 GitHub Actions Concepts + +**Core Concepts to Research:** + +**Workflow Anatomy:** +- `name` - What is your workflow called? +- `on` - When does it run? (push, pull_request, schedule, etc.) +- `jobs` - What work needs to be done? +- `steps` - Individual commands within a job +- `runs-on` - What OS environment? (ubuntu-latest, etc.) + +**Key Questions:** +- Should you run CI on every push, or only on pull requests? +- What happens if tests fail? Should the workflow continue? +- How do you access secrets (like Docker Hub credentials) securely? +- Why might you want multiple jobs vs multiple steps in one job? + +**Python CI Steps Pattern:** +```yaml +# This is a pattern, not exact copy-paste code +# Research the actual syntax and actions needed + +- Set up Python environment +- Install dependencies +- Run linter +- Run tests +``` + +**Docker CI Steps Pattern:** +```yaml +# This is a pattern, not exact copy-paste code +# Research the actual actions and their parameters + +- Log in to Docker Hub +- Extract metadata for tags +- Build and push Docker image +``` + +**Important Concepts:** +- **Actions Marketplace:** Reusable actions (actions/checkout@v4, actions/setup-python@v5, docker/build-push-action@v6) +- **Secrets:** How to store Docker Hub credentials securely +- **Job Dependencies:** Can one job depend on another succeeding? +- **Matrix Builds:** Testing multiple Python versions (optional but good to know) +- **Caching:** Speed up workflows by caching dependencies (we'll add this in Task 3) + +**Resources:** +- [GitHub Actions Documentation](https://docs.github.com/en/actions) +- [Building and Testing Python](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python) +- [Publishing Docker Images](https://docs.docker.com/ci-cd/github-actions/) +- [GitHub Actions Marketplace](https://github.com/marketplace?type=actions) +- [Workflow Syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) + +**Security Best Practices:** +- Never hardcode passwords or tokens in workflow files +- Use GitHub Secrets for sensitive data +- Understand when secrets are exposed to pull requests from forks +- Use `secrets.GITHUB_TOKEN` for GitHub API access (auto-provided) + +**Docker Hub Authentication:** +You'll need to create a Docker Hub access token and add it as a GitHub Secret. Research: +- How to create Docker Hub access tokens +- How to add secrets to your GitHub repository +- How to reference secrets in workflow files (hint: `${{ secrets.NAME }}`) + +
+ +
+💡 Versioning Strategy Guidance + +**Semantic Versioning (SemVer):** + +**Format:** MAJOR.MINOR.PATCH (e.g., 1.2.3) +- **MAJOR:** Breaking changes (incompatible API changes) +- **MINOR:** New features (backward-compatible) +- **PATCH:** Bug fixes (backward-compatible) + +**Implementation Approaches:** +1. **Manual Git Tags:** Create git tags (v1.0.0) and reference in workflow +2. **Automated from Commits:** Parse conventional commits to bump version +3. **GitHub Releases:** Trigger on release creation + +**Docker Tagging Example:** +- `username/app:1.2.3` (full version) +- `username/app:1.2` (minor version, rolling) +- `username/app:1` (major version, rolling) +- `username/app:latest` (latest stable) + +**Pros:** Clear when breaking changes occur, industry standard for libraries +**Cons:** Requires discipline to follow rules correctly + +--- + +**Calendar Versioning (CalVer):** + +**Common Formats:** +- `YYYY.MM.DD` (e.g., 2024.01.15) - Daily releases +- `YYYY.MM.MICRO` (e.g., 2024.01.0) - Monthly with patch number +- `YYYY.0M` (e.g., 2024.01) - Monthly releases + +**Implementation Approaches:** +1. **Date-based:** Generate from current date in workflow +2. **Git SHA:** Combine with short commit SHA (2024.01-a1b2c3d) +3. **Build Number:** Use GitHub run number (2024.01.42) + +**Docker Tagging Example:** +- `username/app:2024.01` (month version) +- `username/app:2024.01.123` (with build number) +- `username/app:latest` (latest build) + +**Pros:** No ambiguity, good for continuous deployment, easier to remember +**Cons:** Doesn't indicate breaking changes + +--- + +**How to Implement in CI:** + +**Using docker/metadata-action:** +```yaml +# Pattern - research actual syntax +- name: Docker metadata + uses: docker/metadata-action + with: + # Define your tagging strategy here + # Can reference git tags, dates, commit SHAs +``` + +**Manual Tagging:** +```yaml +# Pattern - research actual syntax +- name: Generate version + run: echo "VERSION=$(date +%Y.%m.%d)" >> $GITHUB_ENV + +- name: Build and push + # Use ${{ env.VERSION }} in tags +``` + +**Questions to Consider:** +- How often will you release? (Daily? Per feature? Monthly?) +- Do users need to know about breaking changes explicitly? +- Are you building a library (use SemVer) or a service (CalVer works)? +- How will you track what's in each version? + +**Resources:** +- [Semantic Versioning](https://semver.org/) +- [Calendar Versioning](https://calver.org/) +- [Docker Metadata Action](https://github.com/docker/metadata-action) +- [Conventional Commits](https://www.conventionalcommits.org/) (for automated SemVer) + +
+ +
+💡 Debugging GitHub Actions + +**Common Issues & How to Debug:** + +**Workflow Won't Trigger:** +- Check your `on:` configuration +- Verify you pushed to the correct branch +- Look at Actions tab for filtering options + +**Steps Failing:** +- Click into the failed step to see full logs +- Check for typos in action names or parameters +- Verify secrets are configured correctly +- Test commands locally first + +**Docker Build Fails:** +- Ensure Dockerfile is in the correct location +- Check context path in build step +- Verify base image exists and is accessible +- Test Docker build locally first + +**Authentication Issues:** +- Verify secret names match exactly (case-sensitive) +- Check that Docker Hub token has write permissions +- Ensure you're using `docker/login-action` correctly + +**Debugging Techniques:** +- Add `run: echo "Debug message"` steps to understand workflow state +- Use `run: env` to see available environment variables +- Check Actions tab for detailed logs +- Enable debug logging (add `ACTIONS_RUNNER_DEBUG` secret = true) + +
+ +**What to Document:** +- Your workflow trigger strategy and reasoning +- Why you chose specific actions from the marketplace +- Your Docker tagging strategy (latest? version tags? commit SHA?) +- Link to successful workflow run in GitHub Actions tab +- Terminal output or screenshot of green checkmark + +--- + +### Task 3 — CI Best Practices & Security (3 pts) + +**Objective:** Optimize your CI workflow and add security scanning. + +**Requirements:** + +1. **Add Status Badge** + - Add a GitHub Actions status badge to your `app_python/README.md` + - The badge should show the current workflow status (passing/failing) + +2. **Implement Dependency Caching** + - Add caching for Python dependencies to speed up workflow + - Measure and document the speed improvement + +3. **Add Security Scanning with Snyk** + - Integrate Snyk vulnerability scanning into your workflow + - Configure it to check for vulnerabilities in your dependencies + - Document any vulnerabilities found and how you addressed them + +4. **Apply CI Best Practices** + - Research and implement at least 3 additional CI best practices + - Document which practices you applied and why they matter + +
+💡 CI Best Practices Guidance + +**Dependency Caching:** + +Caching speeds up workflows by reusing previously downloaded dependencies. + +**Key Concepts:** +- What should be cached? (pip packages, Docker layers, etc.) +- What's the cache key? (based on requirements.txt hash) +- When does cache become invalid? +- How much time does caching save? + +**Actions to Research:** +- `actions/cache` for general caching +- `actions/setup-python` has built-in cache support + +**Questions to Explore:** +- Where are Python packages stored that should be cached? +- How do you measure cache hit vs cache miss? +- What happens if requirements.txt changes? + +**Status Badges:** + +Show workflow status directly in your README. + +**Format Pattern:** +```markdown +![Workflow Name](https://github.com/username/repo/workflows/workflow-name/badge.svg) +``` + +Research how to: +- Get the correct badge URL for your workflow +- Make badges clickable (link to Actions tab) +- Display specific branch status + +**CI Best Practices to Consider:** + +Research and choose at least 3 to implement: + +1. **Fail Fast:** Stop workflow on first failure +2. **Matrix Builds:** Test multiple Python versions (3.12, 3.13) +3. **Job Dependencies:** Don't push Docker if tests fail +4. **Conditional Steps:** Only push on main branch +5. **Pull Request Checks:** Require passing CI before merge +6. **Workflow Concurrency:** Cancel outdated workflow runs +7. **Docker Layer Caching:** Cache Docker build layers +8. **Environment Variables:** Use env for repeated values +9. **Secrets Scanning:** Prevent committing secrets +10. **YAML Validation:** Lint your workflow files + +**Resources:** +- [GitHub Actions Best Practices](https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits) +- [Caching Dependencies](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) +- [Security Hardening](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) + +
+ +
+💡 Snyk Integration Guidance + +**What is Snyk?** + +Snyk is a security tool that scans your dependencies for known vulnerabilities. + +**Key Concepts:** +- Vulnerability databases (CVEs) +- Severity levels (low, medium, high, critical) +- Automated dependency updates +- Security advisories + +**Integration Options:** + +1. **Snyk GitHub Action:** + - Use `snyk/actions` from GitHub Marketplace + - Requires Snyk API token (free tier available) + - Can fail builds on vulnerabilities + +2. **Snyk CLI in Workflow:** + - Install Snyk CLI in workflow + - Run `snyk test` command + - More flexible but requires setup + +**Setup Steps:** +1. Create free Snyk account +2. Get API token from Snyk dashboard +3. Add token as GitHub Secret +4. Add Snyk step to workflow +5. Configure severity threshold (what level fails the build?) + +**Questions to Explore:** +- Should every vulnerability fail your build? +- What if vulnerabilities have no fix available? +- How do you handle false positives? +- When should you break the build vs just warn? + +**Resources:** +- [Snyk GitHub Actions](https://github.com/snyk/actions) +- [Snyk Python Example](https://github.com/snyk/actions/tree/master/python) +- [Snyk Documentation](https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration) + +**Common Issues:** +- Dependencies not installed before Snyk runs +- API token not configured correctly +- Overly strict severity settings breaking builds +- Virtual environment confusion + +**What to Document:** +- Your severity threshold decision and reasoning +- Any vulnerabilities found and your response +- Whether you fail builds on vulnerabilities or just warn + +
+ +**What to Document:** +- Status badge in README (visible proof it works) +- Caching implementation and speed improvement metrics +- CI best practices you applied with explanations +- Snyk integration results and vulnerability handling +- Terminal output showing improved workflow performance + +--- + +## Bonus Task — Multi-App CI with Path Filters + Test Coverage (2.5 pts) + +**Objective:** Set up CI for your compiled language app with intelligent path-based triggers AND add test coverage tracking. + +**Part 1: Multi-App CI (1.5 pts)** + +1. **Create Second CI Workflow** + - Create `.github/workflows/-ci.yml` for your Go/Rust/Java app + - Implement similar CI steps (lint, test, build Docker image) + - Use language-specific actions and best practices + - Apply versioning strategy (SemVer or CalVer) consistently + +2. **Implement Path-Based Triggers** + - Python workflow should only run when `app_python/` files change + - Compiled language workflow should only run when `app_/` files change + - Neither should run when only docs or other files change + +3. **Optimize for Multiple Apps** + - Ensure both workflows can run in parallel + - Consider using workflow templates (DRY principle) + - Document the benefits of path-based triggers + +**Part 2: Test Coverage Badge (1 pt)** + +4. **Add Coverage Tracking** + - Install coverage tool (`pytest-cov` for Python, coverage tool for your other language) + - Generate coverage reports in CI workflow + - Integrate with codecov.io or coveralls.io (free for public repos) + - Add coverage badge to README showing percentage + +5. **Coverage Goals** + - Document your current coverage percentage + - Identify what's not covered and why + - Set a coverage threshold in CI (e.g., fail if below 70%) + +
+💡 Path Filters & Multi-App CI + +**Why Path Filters?** + +In a monorepo with multiple apps, you don't want to run Python CI when only Go code changes. + +**Path Filter Syntax:** +```yaml +on: + push: + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' +``` + +**Key Concepts:** +- Glob patterns for path matching +- When to include workflow file itself +- Exclude patterns (paths-ignore) +- How to test path filters + +**Questions to Explore:** +- Should changes to README.md trigger CI? +- Should changes to the root .gitignore trigger CI? +- What about changes to both apps in one commit? +- How do you test that path filters work correctly? + +**Multi-Language CI Patterns:** + +**For Go:** +- actions/setup-go +- golangci-lint for linting +- go test for testing +- Multi-stage Docker builds (from Lab 2 bonus) + +**For Rust:** +- actions-rs/toolchain +- cargo clippy for linting +- cargo test for testing +- cargo-audit for security + +**For Java:** +- actions/setup-java +- Maven or Gradle for build +- Checkstyle or SpotBugs for linting +- JUnit tests + +**Workflow Reusability:** + +Consider: +- Reusable workflows (call one workflow from another) +- Composite actions (bundle steps together) +- Workflow templates (DRY for similar workflows) + +**Resources:** +- [Path Filters](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onpushpull_requestpaths) +- [Reusable Workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows) +- [Starter Workflows](https://github.com/actions/starter-workflows/tree/main/ci) + +
+ +
+💡 Test Coverage Tracking + +**What is Test Coverage?** + +Coverage measures what percentage of your code is executed by your tests. High coverage = more code is tested. + +**Why Coverage Matters:** +- Identifies untested code paths +- Prevents regressions (changes breaking untested code) +- Increases confidence in refactoring +- Industry standard quality metric + +**Coverage Tools by Language:** + +**Python (pytest-cov):** +```bash +# Install +pip install pytest-cov + +# Run with coverage +pytest --cov=app_python --cov-report=xml --cov-report=term + +# Generates coverage.xml for upload +``` + +**Go (built-in):** +```bash +go test -coverprofile=coverage.out ./... +go tool cover -html=coverage.out +``` + +**Rust (tarpaulin):** +```bash +cargo install cargo-tarpaulin +cargo tarpaulin --out Xml +``` + +**Java (JaCoCo with Maven/Gradle):** +```bash +mvn test jacoco:report +# or +gradle test jacocoTestReport +``` + +**Integration Services:** + +**Codecov (Recommended):** +- Free for public repos +- Beautiful visualizations +- PR comments with coverage diff +- Setup: Sign in with GitHub, add repo, upload coverage report + +**Coveralls:** +- Alternative to Codecov +- Similar features +- Different UI + +**Coverage in CI Workflow:** +```yaml +# Pattern for Python (research actual syntax) +- name: Run tests with coverage + run: pytest --cov=. --cov-report=xml + +- name: Upload to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} +``` + +**Coverage Badge:** +```markdown +![Coverage](https://codecov.io/gh/username/repo/branch/main/graph/badge.svg) +``` + +**Setting Coverage Thresholds:** + +You can fail CI if coverage drops below a threshold: + +```yaml +# In pytest.ini or pyproject.toml +[tool:pytest] +addopts = --cov=. --cov-fail-under=70 +``` + +**Questions to Consider:** +- What's a reasonable coverage target? (70%? 80%? 90%?) +- Should you aim for 100% coverage? (Usually no - diminishing returns) +- What code is OK to leave untested? (Error handlers, config, main) +- How do you test hard-to-reach code paths? + +**Best Practices:** +- Don't chase 100% coverage blindly +- Focus on testing critical business logic +- Integration points should have high coverage +- Simple getters/setters can be skipped +- Measure coverage trends, not just absolute numbers + +**Resources:** +- [Codecov Documentation](https://docs.codecov.com/) +- [pytest-cov Documentation](https://pytest-cov.readthedocs.io/) +- [Go Coverage](https://go.dev/blog/cover) +- [Cargo Tarpaulin](https://github.com/xd009642/tarpaulin) +- [JaCoCo](https://www.jacoco.org/) + +
+ +**What to Document:** +- Second workflow implementation with language-specific best practices +- Path filter configuration and testing proof +- Benefits analysis: Why path filters matter in monorepos +- Example showing workflows running independently +- Terminal output or Actions tab showing selective triggering +- **Coverage integration:** Screenshot/link to codecov/coveralls dashboard +- **Coverage analysis:** Current percentage, what's covered/not covered, your threshold + +--- + +## How to Submit + +1. **Create Branch:** + - Create a new branch called `lab03` + - Develop your CI workflows on this branch + +2. **Commit Work:** + - Add workflow files (`.github/workflows/`) + - Add test files (`app_python/tests/`) + - Add documentation (`app_python/docs/LAB03.md`) + - Commit with descriptive message following conventional commits + +3. **Verify CI Works:** + - Push to your fork and verify workflows run + - Check that all jobs pass + - Review workflow logs for any issues + +4. **Create Pull Requests:** + - **PR #1:** `your-fork:lab03` → `course-repo:master` + - **PR #2:** `your-fork:lab03` → `your-fork:master` + - CI should run automatically on your PRs + +--- + +## Acceptance Criteria + +### Main Tasks (10 points) + +**Unit Testing (3 pts):** +- [ ] Testing framework chosen with justification +- [ ] Tests exist in `app_python/tests/` directory +- [ ] All endpoints have test coverage +- [ ] Tests pass locally (terminal output provided) +- [ ] README updated with testing instructions + +**GitHub Actions CI (4 pts):** +- [ ] Workflow file exists at `.github/workflows/python-ci.yml` +- [ ] Workflow includes: dependency installation, linting, testing +- [ ] Workflow includes: Docker Hub login, build, and push +- [ ] Versioning strategy chosen (SemVer or CalVer) and implemented +- [ ] Docker images tagged with at least 2 tags (e.g., version + latest) +- [ ] Workflow triggers configured appropriately +- [ ] All workflow steps pass successfully +- [ ] Docker Hub shows versioned images +- [ ] Link to successful workflow run provided + +**CI Best Practices (3 pts):** +- [ ] Status badge added to README and working +- [ ] Dependency caching implemented with performance metrics +- [ ] Snyk security scanning integrated +- [ ] At least 3 CI best practices applied +- [ ] Documentation complete (see Documentation Requirements section) + +### Bonus Task (2.5 points) + +**Part 1: Multi-App CI (1.5 pts)** +- [ ] Second workflow created for compiled language app (`.github/workflows/-ci.yml`) +- [ ] Language-specific linting and testing implemented +- [ ] Versioning strategy applied to second app +- [ ] Path filters configured for both workflows +- [ ] Path filters tested and proven to work (workflows run selectively) +- [ ] Both workflows can run in parallel +- [ ] Documentation explains benefits and shows selective triggering + +**Part 2: Test Coverage (1 pt)** +- [ ] Coverage tool integrated (`pytest-cov` or equivalent) +- [ ] Coverage reports generated in CI workflow +- [ ] Codecov or Coveralls integration complete +- [ ] Coverage badge added to README +- [ ] Coverage threshold set in CI (optional but recommended) +- [ ] Documentation includes coverage analysis (percentage, what's covered/not) + +--- + +## Documentation Requirements + +Create `app_python/docs/LAB03.md` with these sections: + +### 1. Overview +- Testing framework used and why you chose it +- What endpoints/functionality your tests cover +- CI workflow trigger configuration (when does it run?) +- Versioning strategy chosen (SemVer or CalVer) and rationale + +### 2. Workflow Evidence +``` +Provide links/terminal output for: +- ✅ Successful workflow run (GitHub Actions link) +- ✅ Tests passing locally (terminal output) +- ✅ Docker image on Docker Hub (link to your image) +- ✅ Status badge working in README +``` + +### 3. Best Practices Implemented +Quick list with one-sentence explanations: +- **Practice 1:** Why it helps +- **Practice 2:** Why it helps +- **Practice 3:** Why it helps +- **Caching:** Time saved (before vs after) +- **Snyk:** Any vulnerabilities found? Your action taken + +### 4. Key Decisions +Answer these briefly (2-3 sentences each): +- **Versioning Strategy:** SemVer or CalVer? Why did you choose it for your app? +- **Docker Tags:** What tags does your CI create? (e.g., latest, version number, etc.) +- **Workflow Triggers:** Why did you choose those triggers? +- **Test Coverage:** What's tested vs not tested? + +### 5. Challenges (Optional) +- Any issues you encountered and how you fixed them +- Keep it brief - bullet points are fine + +--- + +## Rubric + +| Criteria | Points | Description | +|----------|--------|-------------| +| **Unit Testing** | 3 pts | Comprehensive tests, good coverage | +| **CI Workflow** | 4 pts | Complete, functional, automated | +| **Best Practices** | 3 pts | Optimized, secure, well-documented | +| **Bonus** | 2.5 pts | Multi-app CI with path filters | +| **Total** | 12.5 pts | 10 pts required + 2.5 pts bonus | + +**Grading:** +- **10/10:** All tasks complete, CI works flawlessly, clear documentation, meaningful tests +- **8-9/10:** CI works, good test coverage, best practices applied, solid documentation +- **6-7/10:** CI functional, basic tests, some best practices, minimal documentation +- **<6/10:** CI broken or missing steps, poor tests, incomplete work + +**Quick Checklist for Full Points:** +- ✅ Tests actually test your endpoints (not just imports) +- ✅ CI workflow runs and passes +- ✅ Docker image builds and pushes successfully +- ✅ At least 3 best practices applied (caching, Snyk, status badge, etc.) +- ✅ Documentation complete but concise (no essay needed!) +- ✅ Links/evidence provided (workflow runs, Docker Hub, etc.) + +**Documentation Should Take:** 15-30 minutes to write, 5 minutes to review + +--- + +## Resources + +
+📚 GitHub Actions Documentation + +- [GitHub Actions Quickstart](https://docs.github.com/en/actions/quickstart) +- [Workflow Syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) +- [Building and Testing Python](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python) +- [Publishing Docker Images](https://docs.docker.com/ci-cd/github-actions/) +- [GitHub Actions Marketplace](https://github.com/marketplace?type=actions) + +
+ +
+🧪 Testing Resources + +- [Pytest Documentation](https://docs.pytest.org/) +- [Flask Testing Guide](https://flask.palletsprojects.com/en/stable/testing/) +- [FastAPI Testing Guide](https://fastapi.tiangolo.com/tutorial/testing/) +- [Python Testing Best Practices](https://realpython.com/python-testing/) + +
+ +
+🔒 Security & Quality + +- [Snyk GitHub Actions](https://github.com/snyk/actions) +- [Snyk Python Integration](https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration) +- [GitHub Security Best Practices](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) +- [Dependency Scanning](https://docs.github.com/en/code-security/supply-chain-security) + +
+ +
+⚡ Performance & Optimization + +- [Caching Dependencies](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) +- [Docker Build Cache](https://docs.docker.com/build/cache/) +- [Workflow Optimization](https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration) + +
+ +
+🛠️ CI/CD Tools + +- [act](https://github.com/nektos/act) - Run GitHub Actions locally +- [actionlint](https://github.com/rhysd/actionlint) - Lint workflow files +- [GitHub CLI](https://cli.github.com/) - Manage workflows from terminal + +
+ +--- + +## Looking Ahead + +- **Lab 4-6:** CI will validate your Terraform and Ansible code +- **Lab 7-8:** CI will run integration tests with logging/metrics +- **Lab 9-10:** CI will validate Kubernetes manifests and Helm charts +- **Lab 13:** ArgoCD will deploy what CI builds (GitOps!) +- **All Future Labs:** This pipeline is your safety net for changes + +--- + +**Good luck!** 🚀 + +> **Remember:** CI isn't about having green checkmarks—it's about catching problems before they reach production. Focus on meaningful tests and understanding why each practice matters. Think like a DevOps engineer: automate everything, fail fast, and learn from failures. diff --git a/labs/lab04.md b/labs/lab04.md new file mode 100644 index 0000000000..eefa858953 --- /dev/null +++ b/labs/lab04.md @@ -0,0 +1,1510 @@ +# Lab 4 — Infrastructure as Code (Terraform & Pulumi) + +![difficulty](https://img.shields.io/badge/difficulty-beginner-success) +![topic](https://img.shields.io/badge/topic-Infrastructure%20as%20Code-blue) +![points](https://img.shields.io/badge/points-10%2B2.5-orange) +![tech](https://img.shields.io/badge/tech-Terraform%20%7C%20Pulumi-informational) + +> Provision cloud infrastructure using code with Terraform and Pulumi, comparing both approaches. + +## Overview + +Learn Infrastructure as Code (IaC) by creating virtual machines in the cloud using two popular tools: Terraform (declarative, HCL) and Pulumi (imperative, real programming languages). + +**What You'll Learn:** +- Terraform fundamentals and HCL syntax +- Pulumi fundamentals and infrastructure with code +- Cloud provider APIs and resources +- Infrastructure lifecycle management +- IaC best practices and validation +- Comparing IaC tools and approaches + +**Connection to Previous Labs:** +- **Lab 2:** Created Docker images - now we'll provision infrastructure to run them +- **Lab 3:** CI/CD for applications - now we'll add CI/CD for infrastructure +- **Lab 5:** Ansible will provision software on these VMs (you'll need a VM ready!) + +**Tech Stack:** Terraform 1.9+ | Pulumi 3.x | Yandex Cloud / AWS + +**Why Two Tools?** +By using both Terraform and Pulumi for the same task, you'll understand: +- Different IaC philosophies (declarative vs imperative) +- Tool trade-offs and use cases +- How to evaluate IaC tools for your needs + +**Important for Lab 5:** +The VM you create in this lab will be used in **Lab 5 (Ansible)** for configuration management. You have two options: +- **Option A (Recommended):** Keep your cloud VM running until you complete Lab 5 +- **Option B:** Use a local VM (see Local VM Alternative section below) + +If you choose to destroy your cloud VM after Lab 4, you can easily recreate it later using your Terraform/Pulumi code! + +--- + +## Important: Cloud Provider Selection + +### Recommended for Russia: Yandex Cloud + +Yandex Cloud offers free tier and is accessible in Russia: +- 1 VM with 20% vCPU, 1 GB RAM (free tier) +- 10 GB SSD storage +- No credit card required initially + +### Alternative Cloud Providers + +If Yandex Cloud is unavailable, choose any of these: + +**VK Cloud (Russia):** +- Russian cloud provider +- Free trial with bonus credits +- Good documentation in Russian + +**AWS (Amazon Web Services):** +- 750 hours/month free tier (t2.micro) +- Most popular globally +- Extensive documentation + +**GCP (Google Cloud Platform):** +- $300 free credits for 90 days +- Always-free tier for e2-micro +- Modern interface + +**Azure (Microsoft):** +- $200 free credits for 30 days +- Free tier for B1s instances +- Good Windows support + +**DigitalOcean:** +- Simple pricing and interface +- $200 free credits with GitHub Student Pack +- Beginner-friendly + +### Cost Management 🚨 + +**IMPORTANT - Read This:** +- ✅ **Use smallest/free tier instances only** +- ✅ **Run `terraform destroy` when done testing** +- ✅ **Consider keeping VM for Lab 5 to avoid recreation** +- ✅ **Set billing alerts if available** +- ✅ **If not using for Lab 5, delete resources after lab completion** +- ❌ **Never commit cloud credentials to Git** + +--- + +## Local VM Alternative + +If you cannot or prefer not to use cloud providers, you can use a local VM instead. This VM will need to meet specific requirements for Lab 5 (Ansible). + +### Option 1: VirtualBox/VMware VM + +**Requirements:** +- Ubuntu 24.04 LTS (recommended) or Ubuntu 22.04 LTS +- 1 GB RAM minimum (2 GB recommended) +- 10 GB disk space +- Network adapter in Bridged mode (or NAT with port forwarding) +- SSH server installed and configured +- Your SSH public key added to `~/.ssh/authorized_keys` +- Static or predictable IP address + +**Setup Steps:** +```bash +# Install SSH server (if not installed) +sudo apt update +sudo apt install openssh-server + +# Add your SSH public key +mkdir -p ~/.ssh +echo "your-public-key-here" >> ~/.ssh/authorized_keys +chmod 700 ~/.ssh +chmod 600 ~/.ssh/authorized_keys + +# Verify SSH access from your host machine +ssh username@vm-ip-address +``` + +### Option 2: Vagrant VM + +**Requirements:** +- Vagrant installed on your machine +- VirtualBox (or another Vagrant provider) + +**Basic Vagrantfile:** +```ruby +Vagrant.configure("2") do |config| + config.vm.box = "ubuntu/noble64" # Ubuntu 24.04 LTS + # Or use "ubuntu/jammy64" for Ubuntu 22.04 LTS + config.vm.network "private_network", ip: "192.168.56.10" + config.vm.provider "virtualbox" do |vb| + vb.memory = "2048" + end +end +``` + +### Option 3: WSL2 (Windows Subsystem for Linux) + +**Note:** WSL2 can work but has networking limitations. Bridged mode VM is preferred. + +**If using local VM:** +- You can skip Terraform/Pulumi cloud provider setup +- Document your local VM setup instead +- For Task 1, show VM creation (manual or Vagrant) +- For Task 2, you can skip Pulumi (or use Pulumi to manage Vagrant) +- Focus on understanding IaC concepts with cloud provider research + +**Recommended Approach:** +Even with a local VM, complete the Terraform/Pulumi tasks with a cloud provider to gain real IaC experience. You can destroy the cloud VM after Lab 4 and use your local VM for Lab 5. + +--- + +## Tasks + +### Task 1 — Terraform VM Creation (4 pts) + +**Objective:** Create a virtual machine using Terraform on your chosen cloud provider. + +**Requirements:** + +1. **Setup Terraform** + - Install Terraform CLI + - Choose and configure your cloud provider + - Set up authentication (access keys, service accounts, etc.) + - Initialize Terraform + +2. **Define Infrastructure** + + Create a `terraform/` directory with the following resources: + + **Minimum Required Resources:** + - **VM/Compute Instance** (smallest free tier size) + - **Network/VPC** (if required by provider) + - **Security Group/Firewall Rules:** + - Allow SSH (port 22) from your IP + - Allow HTTP (port 80) + - Allow custom port 5000 (for future app deployment) + - **Public IP Address** (to access VM remotely) + +3. **Configuration Best Practices** + - Use variables for configurable values (region, instance type, etc.) + - Use outputs to display important information (public IP, etc.) + - Add appropriate tags/labels for resource identification + - Use `.gitignore` for sensitive files + +4. **Apply Infrastructure** + - Run `terraform plan` to preview changes + - Review the plan carefully + - Apply infrastructure + - Verify VM is accessible via SSH + - Document the public IP and connection method + +5. **State Management** + - Keep state file local (for now) + - Understand what the state file contains + - **Never commit `terraform.tfstate` to Git** + +
+💡 Terraform Fundamentals + +**What is Terraform?** + +Terraform is a declarative IaC tool that lets you define infrastructure in configuration files (HCL - HashiCorp Configuration Language). + +**Key Concepts:** + +**Providers:** +- Plugins that interact with cloud APIs +- Each cloud has its own provider (yandex, aws, google, azurerm) +- Configure authentication and region + +**Resources:** +- Infrastructure components (VMs, networks, firewalls) +- Format: `resource "type" "name" { ... }` +- Each resource has required and optional arguments + +**Data Sources:** +- Query existing infrastructure +- Example: Find latest Ubuntu image ID +- Format: `data "type" "name" { ... }` + +**Variables:** +- Make configurations reusable +- Define in `variables.tf` +- Set values in `terraform.tfvars` (gitignored!) +- Reference: `var.variable_name` + +**Outputs:** +- Display important values after apply +- Example: VM public IP +- Define in `outputs.tf` + +**State File:** +- Tracks real infrastructure +- Maps config to reality +- **Never commit to Git** (contains sensitive data) +- Add to `.gitignore` + +**Typical Workflow:** +```bash +terraform init # Initialize provider plugins +terraform fmt # Format code +terraform validate # Check syntax +terraform plan # Preview changes +terraform apply # Create/update infrastructure +terraform destroy # Delete all infrastructure +``` + +**Resources:** +- [Terraform Documentation](https://developer.hashicorp.com/terraform/docs) +- [Terraform Registry](https://registry.terraform.io/) - Provider docs +- [HCL Syntax](https://developer.hashicorp.com/terraform/language/syntax) + +
+ +
+☁️ Yandex Cloud Terraform Guide + +**Yandex Cloud Setup:** + +**Authentication:** +- Create service account in Yandex Cloud Console +- Generate authorized key (JSON) +- Set key file path or use environment variables + +**Provider Configuration Pattern:** +```hcl +terraform { + required_providers { + yandex = { + source = "yandex-cloud/yandex" + } + } +} + +provider "yandex" { + # Configuration here (zone, folder_id, etc.) +} +``` + +**Key Resources:** +- `yandex_compute_instance` - Virtual machine +- `yandex_vpc_network` - Virtual private cloud +- `yandex_vpc_subnet` - Subnet within VPC +- `yandex_vpc_security_group` - Firewall rules + +**Free Tier Instance:** +- Platform: standard-v2 +- Cores: 2 (core_fraction = 20%) +- Memory: 1 GB +- Boot disk: 10 GB HDD + +**SSH Access:** +- Add SSH public key to `metadata` +- Use `ssh-keys` metadata field +- Connect: `ssh @` + +**Resources:** +- [Yandex Cloud Terraform Provider](https://registry.terraform.io/providers/yandex-cloud/yandex/latest/docs) +- [Getting Started Guide](https://cloud.yandex.com/en/docs/tutorials/infrastructure-management/terraform-quickstart) +- [Compute Instance Example](https://registry.terraform.io/providers/yandex-cloud/yandex/latest/docs/resources/compute_instance) + +
+ +
+☁️ AWS Terraform Guide + +**AWS Setup:** + +**Authentication:** +- Create IAM user with EC2 permissions +- Generate access key ID and secret access key +- Configure AWS CLI or use environment variables +- Never hardcode credentials + +**Provider Configuration Pattern:** +```hcl +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } +} + +provider "aws" { + region = var.region # e.g., "us-east-1" +} +``` + +**Key Resources:** +- `aws_instance` - EC2 instance +- `aws_vpc` - Virtual Private Cloud +- `aws_subnet` - Subnet within VPC +- `aws_security_group` - Firewall rules +- `aws_key_pair` - SSH key + +**Free Tier Instance:** +- Instance type: t2.micro +- AMI: Amazon Linux 2 or Ubuntu (find with data source) +- 750 hours/month free for 12 months +- 30 GB storage included + +**Data Source for AMI:** +Use `aws_ami` data source to find latest Ubuntu image dynamically + +**Resources:** +- [AWS Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) +- [EC2 Instance Resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance) +- [AWS Free Tier](https://aws.amazon.com/free/) + +
+ +
+☁️ GCP Terraform Guide + +**GCP Setup:** + +**Authentication:** +- Create service account in Google Cloud Console +- Download JSON key file +- Set `GOOGLE_APPLICATION_CREDENTIALS` environment variable +- Enable Compute Engine API + +**Provider Configuration Pattern:** +```hcl +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} +``` + +**Key Resources:** +- `google_compute_instance` - VM instance +- `google_compute_network` - VPC network +- `google_compute_subnetwork` - Subnet +- `google_compute_firewall` - Firewall rules + +**Free Tier Instance:** +- Machine type: e2-micro +- Zone: us-central1-a (or other free tier zone) +- Always free (within limits) +- Boot disk: 30 GB standard persistent disk + +**Resources:** +- [Google Provider Documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs) +- [Compute Instance Resource](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) +- [GCP Free Tier](https://cloud.google.com/free) + +
+ +
+☁️ Other Cloud Providers + +**Azure:** +- Provider: `azurerm` +- Resource: `azurerm_linux_virtual_machine` +- Free tier: B1s instance +- [Azure Provider Docs](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs) + +**VK Cloud:** +- Based on OpenStack +- Provider: OpenStack provider +- [VK Cloud Documentation](https://mcs.mail.ru/help/) + +**DigitalOcean:** +- Provider: `digitalocean` +- Resource: `digitalocean_droplet` +- Simple and beginner-friendly +- [DigitalOcean Provider Docs](https://registry.terraform.io/providers/digitalocean/digitalocean/latest/docs) + +**Questions to Explore:** +- What's the smallest instance size for your provider? +- How do you find the right OS image ID? +- What authentication method does your provider use? +- How do you add SSH keys to instances? + +
+ +
+🔒 Security Best Practices + +**Credentials Management:** + +**❌ NEVER DO THIS:** +```hcl +provider "aws" { + access_key = "AKIAIOSFODNN7EXAMPLE" # NEVER! + secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" # NEVER! +} +``` + +**✅ DO THIS INSTEAD:** + +**Option 1: Environment Variables** +```bash +export AWS_ACCESS_KEY_ID="your-key" +export AWS_SECRET_ACCESS_KEY="your-secret" +# Provider will auto-detect +``` + +**Option 2: Credentials File** +```bash +# ~/.aws/credentials (for AWS) +[default] +aws_access_key_id = your-key +aws_secret_access_key = your-secret +``` + +**Option 3: terraform.tfvars (gitignored)** +```hcl +# terraform.tfvars (add to .gitignore!) +access_key = "your-key" +secret_key = "your-secret" +``` + +**Files to Add to .gitignore:** +``` +# Terraform +*.tfstate +*.tfstate.* +.terraform/ +terraform.tfvars +*.tfvars +.terraform.lock.hcl + +# Cloud credentials +*.pem +*.key +*.json # Service account keys +credentials +``` + +**SSH Key Management:** +- Generate SSH key pair locally +- Add public key to cloud provider +- Keep private key secure (never commit) +- Use `chmod 600` on private key file + +**Security Group Rules:** +- Restrict SSH to your IP only (not 0.0.0.0/0) +- Only open ports you need +- Document why each port is open + +
+ +
+📁 Terraform Project Structure + +**Recommended Structure:** + +``` +terraform/ +├── .gitignore # Ignore state, credentials +├── main.tf # Main resources +├── variables.tf # Input variables +├── outputs.tf # Output values +├── terraform.tfvars # Variable values (gitignored!) +└── README.md # Setup instructions +``` + +**What Goes in Each File:** + +**main.tf:** +- Provider configuration +- Resource definitions +- Data sources + +**variables.tf:** +- Variable declarations +- Descriptions +- Default values (non-sensitive only) + +**outputs.tf:** +- Important values to display +- VM IP addresses +- Connection strings + +**terraform.tfvars:** +- Actual variable values +- Secrets and credentials +- **MUST be in .gitignore** + +**Alternative: Single File** +For small projects, you can put everything in `main.tf`, but multi-file is more maintainable. + +
+ +**What to Document:** +- Cloud provider chosen and why +- Terraform version used +- Resources created (VM size, region, etc.) +- Public IP address of created VM +- SSH connection command +- Terminal output from `terraform plan` and `terraform apply` +- Proof of SSH access to VM + +--- + +### Task 2 — Pulumi VM Creation (4 pts) + +**Objective:** Destroy the Terraform VM and recreate the same infrastructure using Pulumi. + +**Requirements:** + +1. **Cleanup Terraform Infrastructure** + - Run `terraform destroy` to delete all resources + - Verify all resources are deleted in cloud console + - Document the cleanup process + +2. **Setup Pulumi** + - Install Pulumi CLI + - Choose a programming language (Python recommended, or TypeScript, Go, C#, Java) + - Initialize a new Pulumi project + - Configure cloud provider + +3. **Recreate Same Infrastructure** + + Create a `pulumi/` directory with equivalent resources: + + **Same Resources as Task 1:** + - VM/Compute Instance (same size) + - Network/VPC + - Security Group/Firewall (same rules) + - Public IP Address + + **Goal:** Functionally identical infrastructure, different tool + +4. **Apply Infrastructure** + - Run `pulumi preview` to see planned changes + - Apply infrastructure with `pulumi up` + - Verify VM is accessible via SSH + - Document the public IP + +5. **Compare Experience** + - What was easier/harder than Terraform? + - How does the code differ? + - Which approach do you prefer and why? + +
+💡 Pulumi Fundamentals + +**What is Pulumi?** + +Pulumi is an imperative IaC tool that lets you write infrastructure using real programming languages (Python, TypeScript, Go, etc.). + +**Key Differences from Terraform:** + +| Aspect | Terraform | Pulumi | +|--------|-----------|--------| +| **Language** | HCL (declarative) | Python, JS, Go, etc. (imperative) | +| **State** | Local or remote state file | Pulumi Cloud (free tier) or self-hosted | +| **Logic** | Limited (count, for_each) | Full programming language | +| **Testing** | External tools | Native unit tests | +| **Secrets** | Plain in state | Encrypted by default | + +**Key Concepts:** + +**Resources:** +- Similar to Terraform, but defined in code +- Example (Python): `vm = compute.Instance("my-vm", ...)` + +**Stacks:** +- Like Terraform workspaces +- Separate environments (dev, staging, prod) +- Each has its own config and state + +**Outputs:** +- Return values from your program +- Example: `pulumi.export("ip", vm.public_ip)` + +**Config:** +- Per-stack configuration +- Set with: `pulumi config set key value` +- Access in code: `config.get("key")` + +**Typical Workflow:** +```bash +pulumi new