diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 368810b..fa0bd72 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,8 +5,13 @@ // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { - "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, - "ghcr.io/devcontainers/features/python:1": {} + "ghcr.io/devcontainers/features/python:1": {}, + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/devcontainers/features/terraform:1": {}, + "ghcr.io/devcontainers-extra/features/ansible:2": {}, + "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {}, + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/devcontainers/features/azure-cli:1": {} }, // Features to add to the dev container. More info: https://containers.dev/features. diff --git a/.gitignore b/.gitignore index 4ac0a20..f157d3c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,46 @@ observability/examples/simple/observability-simple observability/examples/full/cogstack-observability _build +**/.build/ + + +### Terraform Git Ignore +# https://github.com/github/gitignore/blob/main/Terraform.gitignore + +# Local .terraform directories +**/.terraform/ + +# .tfstate files +**/*.tfstate +**/*.tfstate.* + +# Crash log files +**/crash.log +**/crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data +**/*.tfvars +**/*.tfvars.json + +# Ignore override files +**/override.tf +**/override.tf.json +**/*_override.tf +**/*_override.tf.json + +# Ignore transient lock info files created by terraform apply +**/.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +**/.terraformrc +**/terraform.rc + # Python ignores # Byte-compiled / optimized / DLL files diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4af5a38 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "files.exclude": { + "**/.terraform/": true, + "**/.terraform.lock.hcl": true, + "**/terraform.tfstate": true, + "**/terraform.tfstate.backup": true, + "**/terraform.tfstate.*.backup": true, + "**/.ansible/": true + } +} \ No newline at end of file diff --git a/README.md b/README.md index 207f428..0a31e50 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ -# Cogstack Platform Toolkit +# Cogstack Platform This repository contains tools and services used to support CogStack deployments -See the latest documentation on [Readthedocs](https://cogstack-platform-toolkit.readthedocs.io/en/latest/observability/_index.html) +See the latest documentation on [Readthedocs](https://docs.cogstack.org/en/latest/) + +## Project contents +- Source for the official cogstack documentation. This git repo stores the top level documetation that hosts on https://docs.cogstack.org +- CogStack deployment instructions and examples +- CogStack platform tools eg Observability. diff --git a/deployment/terraform/examples/aws-kubernetes/.env.example b/deployment/terraform/examples/aws-kubernetes/.env.example new file mode 100644 index 0000000..f5c5368 --- /dev/null +++ b/deployment/terraform/examples/aws-kubernetes/.env.example @@ -0,0 +1,3 @@ +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_REGION=eu-west-1 \ No newline at end of file diff --git a/deployment/terraform/examples/aws-kubernetes/.gitignore b/deployment/terraform/examples/aws-kubernetes/.gitignore new file mode 100644 index 0000000..1c6e718 --- /dev/null +++ b/deployment/terraform/examples/aws-kubernetes/.gitignore @@ -0,0 +1,2 @@ + +.env \ No newline at end of file diff --git a/deployment/terraform/examples/aws-kubernetes/README.md b/deployment/terraform/examples/aws-kubernetes/README.md new file mode 100644 index 0000000..ae17bc3 --- /dev/null +++ b/deployment/terraform/examples/aws-kubernetes/README.md @@ -0,0 +1,101 @@ +# AWS EKS Deployment + +This is an example deployment of CogStack in AWS. It will create publically accessible services, so is not suitable for production deployment. + +The recommended deployment in AWS is based on using Kubernetes through AWS EKS. + +This example will create a AWS EKS cluster, setup any necessary config, deploy CogStack to the cluster, and test that it is available. + +## Usage +Deployment through terraform is carried out through two terraform commands, to handle the sequencing issues between making a k8s cluster and using it in AWS. + +### Requirements +- Terraform - [Install Terraform](https://developer.hashicorp.com/terraform/install) +- AWS Credentials for an account that can create and destroy resources. + + +### Steps + +### 1. Add Required Secrets for your env +This readme uses environment variables for access: + +1. See the `.env.example` file for the required details. +2. Create a file `.env` with those fields set for your account. +3. Execute `source .env` to set those environment variables + +If desired, see the official documentation for other ways to provide AWS credentials https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration + +### 2. Run Terraform +Terraform is run on two modules for AWS, so we will run one terraform apply in one folder, then another terraform apply in a second folder. + +Initial provisioning takes around 15 minutes. + +```bash +# Set AWS credentials +source .env + +# Create AWS EKS infra +cd eks-cluster +terraform init +terraform apply --auto-approve + +AWS_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# Deploy services to kubernetes +cd ../kubernetes-deployment +export TF_VAR_kubeconfig_file=$AWS_KUBECONFIG +terraform init +terraform apply --auto-approve +``` + +### 3. Accessing the CogStack Platform + +Once the deployment is complete and all services are running, you can access the CogStack platform and its components using the following URLs: + +```bash +terraform output service_urls +``` + + +### Optional - Destroy + +You can destroy the infra to save costs when it wont be used for a long time. + +Do note that there is an initial cost every time the EKS infrastructure is created, looks to be around $0.50 at time of writing. + +```bash +cd ../kubernetes-deployment +terraform destroy + +cd ../eks-cluster +terraform destroy +``` + + +## Optionally use the K8s cluster as normal with the CLI +After setting up the cluster, it is possible to interact directly with it using the kubectl CLI + +The requirement is to get the KUBECONFIG file created by the terraform apply. + +```bash +# Get KUBECONFIG +cd eks-cluster +AWS_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# SET KUBECONFIG +export KUBECONFIG=${AWS_KUBECONFIG} +``` + +Note - alternatively you could use the AWS CLI to set your kubeconfig using `aws eks update-kubeconfig --name $(terraform output -raw cluster_name)`. + +You can then interact with kubernetes via the CLI + +```bash +# Run Medcat service +helm install my-medcat oci://registry-1.docker.io/cogstacksystems/medcat-service-helm --wait --timeout 10m0s + +# Create the ingress +kubectl apply -f resources/ingress-medcat-service.yaml +# Find public url +kubectl get ingress +``` \ No newline at end of file diff --git a/deployment/terraform/examples/aws-kubernetes/eks-cluster/.terraform.lock.hcl b/deployment/terraform/examples/aws-kubernetes/eks-cluster/.terraform.lock.hcl new file mode 100644 index 0000000..ce1fb33 --- /dev/null +++ b/deployment/terraform/examples/aws-kubernetes/eks-cluster/.terraform.lock.hcl @@ -0,0 +1,125 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.0.0" + constraints = ">= 6.0.0, ~> 6.0.0" + hashes = [ + "h1:DhxTTyLjzmC0wRCodYOWkIGwHAYb2Fcg3zFc/3sIpvk=", + "zh:16b1bb786719b7ebcddba3ab751b976ebf4006f7144afeebcb83f0c5f41f8eb9", + "zh:1fbc08b817b9eaf45a2b72ccba59f4ea19e7fcf017be29f5a9552b623eccc5bc", + "zh:304f58f3333dbe846cfbdfc2227e6ed77041ceea33b6183972f3f8ab51bd065f", + "zh:4cd447b5c24f14553bd6e1a0e4fea3c7d7b218cbb2316a3d93f1c5cb562c181b", + "zh:589472b56be8277558616075fc5480fcd812ba6dc70e8979375fc6d8750f83ef", + "zh:5d78484ba43c26f1ef6067c4150550b06fd39c5d4bfb790f92c4a6f7d9d0201b", + "zh:5f470ce664bffb22ace736643d2abe7ad45858022b652143bcd02d71d38d4e42", + "zh:7a9cbb947aaab8c885096bce5da22838ca482196cf7d04ffb8bdf7fd28003e47", + "zh:854df3e4c50675e727705a0eaa4f8d42ccd7df6a5efa2456f0205a9901ace019", + "zh:87162c0f47b1260f5969679dccb246cb528f27f01229d02fd30a8e2f9869ba2c", + "zh:9a145404d506b52078cd7060e6cbb83f8fc7953f3f63a5e7137d41f69d6317a3", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a4eab2649f5afe06cc406ce2aaf9fd44dcf311123f48d344c255e93454c08921", + "zh:bea09141c6186a3e133413ae3a2e3d1aaf4f43466a6a468827287527edf21710", + "zh:d7ea2a35ff55ddfe639ab3b04331556b772a8698eca01f5d74151615d9f336db", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.7" + constraints = ">= 2.0.0" + hashes = [ + "h1:iZ27qylcH/2bs685LJTKOKcQ+g7cF3VwN3kHMrzm4Ow=", + "zh:06f1c54e919425c3139f8aeb8fcf9bceca7e560d48c9f0c1e3bb0a8ad9d9da1e", + "zh:0e1e4cf6fd98b019e764c28586a386dc136129fef50af8c7165a067e7e4a31d5", + "zh:1871f4337c7c57287d4d67396f633d224b8938708b772abfc664d1f80bd67edd", + "zh:2b9269d91b742a71b2248439d5e9824f0447e6d261bfb86a8a88528609b136d1", + "zh:3d8ae039af21426072c66d6a59a467d51f2d9189b8198616888c1b7fc42addc7", + "zh:3ef4e2db5bcf3e2d915921adced43929214e0946a6fb11793085d9a48995ae01", + "zh:42ae54381147437c83cbb8790cc68935d71b6357728a154109d3220b1beb4dc9", + "zh:4496b362605ae4cbc9ef7995d102351e2fe311897586ffc7a4a262ccca0c782a", + "zh:652a2401257a12706d32842f66dac05a735693abcb3e6517d6b5e2573729ba13", + "zh:7406c30806f5979eaed5f50c548eced2ea18ea121e01801d2f0d4d87a04f6a14", + "zh:7848429fd5a5bcf35f6fee8487df0fb64b09ec071330f3ff240c0343fe2a5224", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.4" + constraints = ">= 3.0.0" + hashes = [ + "h1:hkf5w5B6q8e2A42ND2CjAvgvSN3puAosDmOJb3zCVQM=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.2" + constraints = "~> 3.0" + hashes = [ + "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.13.1" + constraints = ">= 0.9.0" + hashes = [ + "h1:+W+DMrVoVnoXo3f3M4W+OpZbkCrUn6PnqDF33D2Cuf0=", + "zh:02cb9aab1002f0f2a94a4f85acec8893297dc75915f7404c165983f720a54b74", + "zh:04429b2b31a492d19e5ecf999b116d396dac0b24bba0d0fb19ecaefe193fdb8f", + "zh:26f8e51bb7c275c404ba6028c1b530312066009194db721a8427a7bc5cdbc83a", + "zh:772ff8dbdbef968651ab3ae76d04afd355c32f8a868d03244db3f8496e462690", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:898db5d2b6bd6ca5457dccb52eedbc7c5b1a71e4a4658381bcbb38cedbbda328", + "zh:8de913bf09a3fa7bedc29fec18c47c571d0c7a3d0644322c46f3aa648cf30cd8", + "zh:9402102c86a87bdfe7e501ffbb9c685c32bbcefcfcf897fd7d53df414c36877b", + "zh:b18b9bb1726bb8cfbefc0a29cf3657c82578001f514bcf4c079839b6776c47f0", + "zh:b9d31fdc4faecb909d7c5ce41d2479dd0536862a963df434be4b16e8e4edc94d", + "zh:c951e9f39cca3446c060bd63933ebb89cedde9523904813973fbc3d11863ba75", + "zh:e5b773c0d07e962291be0e9b413c7a22c044b8c7b58c76e8aa91d1659990dfb5", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.1.0" + constraints = ">= 4.0.0" + hashes = [ + "h1:Ka8mEwRFXBabR33iN/WTIEW6RP0z13vFsDlwn11Pf2I=", + "zh:14c35d89307988c835a7f8e26f1b83ce771e5f9b41e407f86a644c0152089ac2", + "zh:2fb9fe7a8b5afdbd3e903acb6776ef1be3f2e587fb236a8c60f11a9fa165faa8", + "zh:35808142ef850c0c60dd93dc06b95c747720ed2c40c89031781165f0c2baa2fc", + "zh:35b5dc95bc75f0b3b9c5ce54d4d7600c1ebc96fbb8dfca174536e8bf103c8cdc", + "zh:38aa27c6a6c98f1712aa5cc30011884dc4b128b4073a4a27883374bfa3ec9fac", + "zh:51fb247e3a2e88f0047cb97bb9df7c228254a3b3021c5534e4563b4007e6f882", + "zh:62b981ce491e38d892ba6364d1d0cdaadcee37cc218590e07b310b1dfa34be2d", + "zh:bc8e47efc611924a79f947ce072a9ad698f311d4a60d0b4dfff6758c912b7298", + "zh:c149508bd131765d1bc085c75a870abb314ff5a6d7f5ac1035a8892d686b6297", + "zh:d38d40783503d278b63858978d40e07ac48123a2925e1a6b47e62179c046f87a", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb07f708e3316615f6d218cec198504984c0ce7000b9f1eebff7516e384f4b54", + ] +} diff --git a/deployment/terraform/examples/aws-kubernetes/eks-cluster/eks.tf b/deployment/terraform/examples/aws-kubernetes/eks-cluster/eks.tf new file mode 100644 index 0000000..17b682a --- /dev/null +++ b/deployment/terraform/examples/aws-kubernetes/eks-cluster/eks.tf @@ -0,0 +1,79 @@ +# Portions of this code adapted from tha Amazon AWS terraform-aws-eks example module (Apache 2.0): +# https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v21.0.4/examples/eks-auto-mode/README.md#module_eks + +data "aws_availability_zones" "available" { + # Exclude local zones + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +locals { + name = "ex-${basename(path.cwd)}" + kubernetes_version = "1.33" + region = "eu-west-1" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Test = local.name + GithubRepo = "cogstack-devops" + GithubOrg = "CogStack" + } +} + +################################################################################ +# EKS Module +################################################################################ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "21.0.4" + name = local.name + kubernetes_version = local.kubernetes_version + endpoint_public_access = true + + enable_cluster_creator_admin_permissions = true + + compute_config = { + enabled = true + node_pools = ["general-purpose"] + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 6.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} \ No newline at end of file diff --git a/deployment/terraform/examples/aws-kubernetes/eks-cluster/main.tf b/deployment/terraform/examples/aws-kubernetes/eks-cluster/main.tf new file mode 100644 index 0000000..9dcdb60 --- /dev/null +++ b/deployment/terraform/examples/aws-kubernetes/eks-cluster/main.tf @@ -0,0 +1,11 @@ + +resource "null_resource" "copy_kubeconfig" { + depends_on = [module.eks, module.vpc] + + provisioner "local-exec" { + # Extract the kubeconfig file using the AWS CLI. Save it as a local file + command = < v.ansible_playbook_stdout } +# } +# output "ansible_err"{ +# value ={ for k,v in ansible_playbook.playbook : k => v.ansible_playbook_stderr } +# } diff --git a/deployment/terraform/modules/cogstack-docker-services/ansible/deploy-internal.yml b/deployment/terraform/modules/cogstack-docker-services/ansible/deploy-internal.yml new file mode 100644 index 0000000..1c1d7d2 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/ansible/deploy-internal.yml @@ -0,0 +1,27 @@ +- name: Distribute configuration files + hosts: "all" + vars: + LOCAL_SOURCE_DIRECTORY: "{{ playbook_dir }}/../resources" # this is the /deployment directory in this project + DEPLOY_DIRECTORY: /home/ubuntu/cogstack/deployment + tasks: + - name: Create config directory + ansible.builtin.file: + path: "{{ DEPLOY_DIRECTORY }}" + state: directory + mode: "0755" + - name: Copy all configs to the VM + ansible.posix.synchronize: + src: "{{ LOCAL_SOURCE_DIRECTORY }}/config/" + dest: "{{ DEPLOY_DIRECTORY }}/config" + delete: true + recursive: yes + - name: Copy and template prober file with new addresses + ansible.builtin.template: + src: "{{ LOCAL_SOURCE_DIRECTORY }}/config/observability/alloy/probers/probe.cogstack.yml.tmpl" + dest: "{{ DEPLOY_DIRECTORY }}/config/observability/alloy/probers/probe.cogstack.yml" + mode: '0644' + - name: Copy and template observability prober file with new addresses + ansible.builtin.template: + src: "{{ LOCAL_SOURCE_DIRECTORY }}/config/observability/alloy/probers/probe.observability.yml.tmpl" + dest: "{{ DEPLOY_DIRECTORY }}/config/observability/alloy/probers/probe.observability.yml" + mode: '0644' \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/checks.tf b/deployment/terraform/modules/cogstack-docker-services/checks.tf new file mode 100644 index 0000000..fd49ae4 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/checks.tf @@ -0,0 +1,17 @@ + +check "health_check" { + data "http" "medcat_service" { + url = "${local.service_urls.medcat_service}/api/info" + depends_on = [portainer_stack.medcat_service] + } + + assert { + condition = data.http.medcat_service.status_code == 200 + error_message = "${data.http.medcat_service.url} returned an unhealthy status code" + } + + assert { + condition = jsondecode(data.http.medcat_service.response_body).service_app_name == "MedCAT" + error_message = "${data.http.medcat_service.url} returned an unexpected response format" + } +} \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/environments.tf b/deployment/terraform/modules/cogstack-docker-services/environments.tf new file mode 100644 index 0000000..44e973d --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/environments.tf @@ -0,0 +1,10 @@ + + +resource "portainer_environment" "portainer_envs" { + depends_on = [ansible_playbook.playbook] # Add dependency to await the service + for_each = var.hosts + name = each.key + environment_address = "tcp://${each.value.ip_address}:9001" + type = 2 + +} diff --git a/deployment/terraform/modules/cogstack-docker-services/observability.tf b/deployment/terraform/modules/cogstack-docker-services/observability.tf new file mode 100644 index 0000000..c3fb48f --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/observability.tf @@ -0,0 +1,54 @@ +resource "portainer_stack" "observability_stack" { + name = "cogstack-observability" + deployment_type = "standalone" + method = "string" + endpoint_id = local.environments[var.service_targets.observability.hostname] + pull_image = true + + stack_file_content = file("${path.module}/resources/config/observability/observability.docker-compose.yml") + prune = true + env { + name = "BASE_DIR" + value = local.deployed_config_dir + } + env { + name = "CONFIG_HASH" + value = local.config_hashes_by_folder["observability"] + } + env { + name = "ALLOY_HOSTNAME" + value = var.service_targets.observability.hostname + } + env { + name = "ALLOY_IP_ADDRESS" + value = var.hosts[var.service_targets.observability.hostname].ip_address + } +} + +locals { + prometheus_url = "http://${var.hosts[var.service_targets.observability.hostname].ip_address}/prometheus" +} +resource "portainer_stack" "observability_exporters_stack" { + for_each = { for k, v in var.hosts : k => v if !contains([var.service_targets.observability.hostname], v.name) } + + name = "cogstack-observability-exporters" + deployment_type = "standalone" + method = "string" + endpoint_id = local.environments[each.value.name] + pull_image = true + prune = true + stack_file_content = file("${path.module}/resources/config/observability/exporters.docker-compose.yml") + + env { + name = "PROMETHEUS_URL" + value = local.prometheus_url + } + env { + name = "ALLOY_HOSTNAME" + value = each.value.name + } + env { + name = "ALLOY_IP_ADDRESS" + value = each.value.ip_address + } +} diff --git a/deployment/terraform/modules/cogstack-docker-services/outputs.tf b/deployment/terraform/modules/cogstack-docker-services/outputs.tf new file mode 100644 index 0000000..3908c7e --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/outputs.tf @@ -0,0 +1,7 @@ + + +output "service_urls" { + value = local.service_urls + description = "Public URls to call services on " + +} \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/providers.tf b/deployment/terraform/modules/cogstack-docker-services/providers.tf new file mode 100644 index 0000000..d77e9e3 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/providers.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + portainer = { + source = "portainer/portainer" + version = "1.4.2" + } + ansible = { + version = "~> 1.3.0" + source = "ansible/ansible" + } + } +} \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/elasticsearch.alloy b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/elasticsearch.alloy new file mode 100644 index 0000000..0084479 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/elasticsearch.alloy @@ -0,0 +1,29 @@ + +prometheus.exporter.elasticsearch "elasticsearch" { + address = sys.env("ELASTICSEARCH_URL") + basic_auth { + username = sys.env("ELASTICSEARCH_USERNAME") + password = sys.env("ELASTICSEARCH_PASSWORD") + } + ssl_skip_verify = true + +} +discovery.relabel "elasticsearch" { + targets = prometheus.exporter.elasticsearch.elasticsearch.targets + + rule { + target_label = "cluster" + replacement = "elasticsearch-cogstack-cluster" + } + rule { + target_label = "host" + replacement = "elasticsearch_host" + } +} + +prometheus.scrape "elasticsearch_exporter" { + scrape_interval = "15s" + targets = discovery.relabel.elasticsearch.output + forward_to = [prometheus.remote_write.default.receiver] + +} diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/probers/probe.cogstack.yml.tmpl b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/probers/probe.cogstack.yml.tmpl new file mode 100644 index 0000000..8b76218 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/probers/probe.cogstack.yml.tmpl @@ -0,0 +1,9 @@ +# Probe Targets for the new cogstack environment + +- targets: + - "http://{{ MEDCAT_SERVICE_IP }}:5000/api/info" + labels: + name: medcat_service + ip_address: "{{ MEDCAT_SERVICE_IP }}" + host: "{{ MEDCAT_SERVICE_HOSTNAME }}" + job: probe-cogstack diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl new file mode 100644 index 0000000..a718dc6 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl @@ -0,0 +1,14 @@ +- targets: + - cogstack-observability-traefik-1/grafana/api/health + labels: + name: grafana + job: probe-observability-stack + ip_address: "{{ OBSERVABILITY_SERVICE_IP }}" + host: "{{ OBSERVABILITY_SERVICE_HOSTNAME }}" +- targets: + - cogstack-observability-traefik-1/prometheus/-/healthy + labels: + name: prometheus + job: probe-observability-stack + ip_address: "{{ OBSERVABILITY_SERVICE_IP }}" + host: "{{ OBSERVABILITY_SERVICE_HOSTNAME }}" \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/exporters.docker-compose.yml b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/exporters.docker-compose.yml new file mode 100755 index 0000000..6d0bdee --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/exporters.docker-compose.yml @@ -0,0 +1,21 @@ +# Run Exporters on every VM you want metrics from. Grafana Alloy provides multiple components for this +name: "cogstack-observability-exporters" +services: + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest + volumes: + # CAdvisor + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - ${DOCKER_LIB_PATH-/var/lib/docker/}:/var/lib/docker:ro + environment: + - PROMETHEUS_URL=${PROMETHEUS_URL} + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics + networks: + - observability-exporters + +networks: + observability-exporters: + driver: bridge \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/exporters.elasticsearch.docker-compose.yml b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/exporters.elasticsearch.docker-compose.yml new file mode 100644 index 0000000..939f112 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/exporters.elasticsearch.docker-compose.yml @@ -0,0 +1,25 @@ +# Run Exporters on every VM you want metrics from. Grafana Alloy provides multiple components for this +name: "cogstack-observability-exporters" +services: + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest + volumes: + - ${BASE_DIR-.}/observability/alloy/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter + # CAdvisor + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - ${DOCKER_LIB_PATH-/var/lib/docker/}:/var/lib/docker:ro + environment: + - PROMETHEUS_URL=${PROMETHEUS_URL} + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics + - ELASTICSEARCH_URL=${ELASTICSEARCH_URL-https://elasticsearch-1:9200} + - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME-user} # Used to get metrics from Elasticsearch + - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD-pass} # Used to get metrics from Elasticsearch + networks: + - observability-exporters + +networks: + observability-exporters: + driver: bridge \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml new file mode 100644 index 0000000..ced2c3b --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml @@ -0,0 +1,61 @@ +# Observability main stack. Prometheus and Grafana. +# Depends on docker-compose.exporters.yml for the network +name: "cogstack-observability" +services: + prometheus: + image: cogstacksystems/cogstack-observability-prometheus:latest + restart: unless-stopped + volumes: + - ${BASE_DIR-.}/observability/prometheus:/etc/prometheus/cogstack/site + - prometheus-data:/prometheus + networks: + - observability + grafana: + image: cogstacksystems/cogstack-observability-grafana:latest + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafana + networks: + - observability + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + - CONFIG_HASH=${CONFIG_HASH} + traefik: + image: cogstacksystems/cogstack-observability-traefik:latest + networks: + - observability + restart: unless-stopped + ports: + # The HTTP port + - "80:80" + # The Web UI (enabled by --api.insecure=true) + - "8080:8080" + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock:ro + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest + networks: + - observability + ports: + - "12345:12345" + volumes: + - ${BASE_DIR-.}/observability/alloy/probers:/etc/alloy/probers + # CAdvisor + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + environment: + - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics + +networks: + observability: + driver: bridge + +volumes: + prometheus-data: + grafana-data: diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/prometheus/recording-rules/slo.yml b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/prometheus/recording-rules/slo.yml new file mode 100644 index 0000000..a8fbba7 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/config/observability/prometheus/recording-rules/slo.yml @@ -0,0 +1,8 @@ +groups: + - name: slo-target-rules + rules: + # What SLO am I targeting + - record: slo_target_over_30_days + expr: 0.95 # We target 95% uptime over 30 days + labels: + job: "probe-cogstack" #Job here must match the job in the probe targets \ No newline at end of file diff --git a/deployment/terraform/modules/cogstack-docker-services/resources/medcat-service.docker-compose.yml b/deployment/terraform/modules/cogstack-docker-services/resources/medcat-service.docker-compose.yml new file mode 100644 index 0000000..81f3ac0 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/resources/medcat-service.docker-compose.yml @@ -0,0 +1,14 @@ +name: cogstack-medcat-service +services: + medcat-service: + image: cogstacksystems/medcat-service:latest + restart: unless-stopped + environment: + ${ENVIRONMENT_VARIABLES} + ports: + - "5000:5000" + volumes: + - medcat-models:/cat/models/downloaded/ + +volumes: + medcat-models: diff --git a/deployment/terraform/modules/cogstack-docker-services/services.tf b/deployment/terraform/modules/cogstack-docker-services/services.tf new file mode 100644 index 0000000..d26a8f9 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/services.tf @@ -0,0 +1,14 @@ + +resource "portainer_stack" "medcat_service" { + name = "medcat-service" + endpoint_id = local.environments[var.service_targets.medcat_service.hostname] + stack_file_content = templatefile("${path.module}/resources/medcat-service.docker-compose.yml", + { + ENVIRONMENT_VARIABLES = yamlencode(var.service_targets.medcat_service.environment_variables) + }) + deployment_type = "standalone" + method = "string" + pull_image = true + prune = true +} + diff --git a/deployment/terraform/modules/cogstack-docker-services/shared-locals.tf b/deployment/terraform/modules/cogstack-docker-services/shared-locals.tf new file mode 100644 index 0000000..e05fe42 --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/shared-locals.tf @@ -0,0 +1,37 @@ + +locals { + environments = { for vm_name, vm in var.hosts : vm_name => portainer_environment.portainer_envs[vm_name].id } +} + + +locals { + + service_urls = { + grafana = "http://${var.hosts[var.service_targets.observability.hostname].ip_address}/grafana" + prometheus = "http://${var.hosts[var.service_targets.observability.hostname].ip_address}/prometheus" + medcat_service = "http://${var.hosts[var.service_targets.medcat_service.hostname].ip_address}:5000" + } + +} + +locals { + # Setup config file hashing to trigger restarts when config files change. Requires the env var to also be set in the docker-compose.yml files + config_files = fileset("${path.module}/resources/config", "**") # All files + config_hashes = [for f in local.config_files : filemd5("${path.module}/resources/config/${f}")] + hash_of_all_config_files = md5(join("", local.config_hashes)) # One final hash + + + # Setup a hash by config folder, allowing a subset of files to be used to trigger restarts on change. Creates a map like { observability = "12d67ff2e637a4aa077c64803afa457d"} + config_folders = ["observability"] + config_files_by_folder = { + for folder in local.config_folders : + folder => fileset("${path.module}/resources/config/${folder}", "**") + } + config_hashes_by_folder = { + for folder, files in local.config_files_by_folder : + folder => md5(join("", [for f in files : filemd5("${path.module}/resources/config/${folder}/${f}")])) + } + + deployed_config_dir = "/home/ubuntu/cogstack/deployment/config" +} + diff --git a/deployment/terraform/modules/cogstack-docker-services/variables.tf b/deployment/terraform/modules/cogstack-docker-services/variables.tf new file mode 100644 index 0000000..c799e5d --- /dev/null +++ b/deployment/terraform/modules/cogstack-docker-services/variables.tf @@ -0,0 +1,43 @@ + +variable "hosts" { + type = map(object({ + ip_address = string, + unique_name = string, + name = string + })) + description = "Created Hosts: A map of { hostname: { data } }" +} + +variable "ssh_private_key_file" { + type = string + description = "A filepath to a SSH Private key that is used to SSH login to created hosts" +} + +variable "service_targets" { + type = object({ + medcat_service = object({ + hostname = string, + environment_variables = optional(list(string), ["APP_MEDCAT_MODEL_PACK=/cat/models/examples/example-medcat-v1-model-pack.zip"]) + }) + observability = object({ + hostname = string + }) + }) + description = "Target Hosts: A map of { service_name: {hostname: host, environment_variables [ 'some_var: some value' ]} }. The hostname must be a host passed in the hosts variable" + validation { + condition = contains(keys(var.hosts), var.service_targets.medcat_service.hostname) && contains(keys(var.hosts), var.service_targets.observability.hostname) + error_message = "The hostname put here must also be passed in as a key in the hosts variable" + } +} + +variable "portainer_secrets" { + type = object({ + api_key = string, + }) + + description = < /dev/null + - sudo apt-get update + - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + +# Run Portainer + - docker pull portainer/portainer-ce:lts + - docker network create portainer-network + - docker volume create portainer-data + - | + docker run -d \ + --name portainer \ + --restart unless-stopped \ + --network portainer-network \ + -p 9443:9443 \ + -e AGENT_SECRET="${PORTAINER_AGENT_SECRET}" \ + -v portainer-data:/data \ + -l 'traefik.enable="true"' \ + -l 'traefik.http.routers.portainer-path-router.rule="PathPrefix(`/portainer`)"' \ + portainer/portainer-ce:lts + - docker pull portainer/agent:latest + - | + docker run -d \ + --name portainer_agent \ + -p 9001:9001 \ + -e AGENT_SECRET=${PORTAINER_AGENT_SECRET} \ + --network portainer-network \ + --restart unless-stopped \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -l 'traefik.enable="true"' \ + -l 'traefik.http.routers.portainer-path-router.rule="PathPrefix(`/portainer-agent`)"' \ + portainer/agent:latest + + - PORTAINER_URL=https://localhost:9443 + + - INIT_FILE=/opt/cogstack/init/portainer-init-snapshot.tar.gz + + - | + curl --insecure --request POST \ + --url $${PORTAINER_URL}/api/restore \ + --header 'Content-Type: multipart/form-data' \ + --form file=@$${INIT_FILE} \ + --form fileName=backup \ + --form password=${PORTAINER_SNAPSHOT_PASSWORD} + \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-cogstack-infra/cloud-init.yaml b/deployment/terraform/modules/openstack-cogstack-infra/cloud-init.yaml new file mode 100644 index 0000000..f654502 --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/cloud-init.yaml @@ -0,0 +1,43 @@ +#cloud-config +# create the docker group +groups: + - docker + +# Add default auto created user to docker group +system_info: + default_user: + groups: [docker] + +runcmd: +# Install Docker + # Add Docker's official GPG key: + - sudo apt-get update + - sudo apt-get install ca-certificates curl + - sudo install -m 0755 -d /etc/apt/keyrings + - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + - sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + - | + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + - sudo apt-get update + - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + +# Run Portainer + - docker network create portainer-network + - docker pull portainer/agent:latest + - | + docker run -d \ + --name portainer_agent \ + -p 9001:9001 \ + -e AGENT_SECRET=${PORTAINER_AGENT_SECRET} \ + --network portainer-network \ + --restart unless-stopped \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -l 'traefik.enable="true"' \ + -l 'traefik.http.routers.portainer-path-router.rule="PathPrefix(`/portainer-agent`)"' \ + portainer/agent:latest diff --git a/deployment/terraform/modules/openstack-cogstack-infra/compute-keypair.tf b/deployment/terraform/modules/openstack-cogstack-infra/compute-keypair.tf new file mode 100644 index 0000000..819bd71 --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/compute-keypair.tf @@ -0,0 +1,33 @@ +# Create a Keypair resource in openstack +# Either generate a brand new public and private key, when the var.ssh_key_pair variable is not provided +# Or use the file contents in var.ssh_key_pair + +locals { + is_using_existing_ssh_keypair = var.ssh_key_pair != null + ssh_keys = { + # If user inputs an existing SSH keypair, then pass it through. Else use the generated resources. + private_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.private_key_file) : openstack_compute_keypair_v2.compute_keypair.private_key + private_key_file = local.is_using_existing_ssh_keypair ? var.ssh_key_pair.private_key_file : abspath(local_file.private_key[0].filename) + public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : openstack_compute_keypair_v2.compute_keypair.public_key + public_key_file = local.is_using_existing_ssh_keypair ? var.ssh_key_pair.public_key_file : abspath(local_file.public_key[0].filename) + } +} + +resource "openstack_compute_keypair_v2" "compute_keypair" { + name = "${local.random_prefix}-cogstack_keypair" + public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : null +} + +resource "local_file" "private_key" { + count = local.is_using_existing_ssh_keypair ? 0 : 1 + content = openstack_compute_keypair_v2.compute_keypair.private_key + filename = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pem" + file_permission = "0600" +} + +resource "local_file" "public_key" { + count = local.is_using_existing_ssh_keypair ? 0 : 1 + content = openstack_compute_keypair_v2.compute_keypair.public_key + filename = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub" + file_permission = "0600" +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-cogstack-infra/compute.tf b/deployment/terraform/modules/openstack-cogstack-infra/compute.tf new file mode 100644 index 0000000..2ee258f --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/compute.tf @@ -0,0 +1,94 @@ + +resource "openstack_compute_instance_v2" "cogstack_ops_compute" { + depends_on = [openstack_networking_secgroup_rule_v2.cogstack_apps_port_rules["22-SSH"]] + for_each = { for vm in var.host_instances : vm.name => vm } + name = "${local.random_prefix}-${each.value.name}" + flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id + key_pair = openstack_compute_keypair_v2.compute_keypair.name + region = "RegionOne" + + user_data = each.value.is_controller ? data.cloudinit_config.init_docker_controller.rendered : data.cloudinit_config.init_docker.rendered + security_groups = ["default", + openstack_networking_secgroup_v2.cogstack_apps_security_group.name + ] + + network { + uuid = data.openstack_networking_network_v2.external_4003.id + } + + block_device { + uuid = data.openstack_images_image_v2.ubuntu.id + source_type = "image" + volume_size = each.value.volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + connection { + user = "ubuntu" + host = self.access_ip_v4 + private_key = file(local.ssh_keys.private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait > /tmp/openstack-terraform-remote-exec-provisioner.log || true", + ] + } + +} + +# TODO: Read content from files and put into cloud-init config +# data "local_file" "install_docker_sh" { +# filename = "${path.module}/resources/install-docker.sh" +# } + +# data "local_file" "initialize_portainer_binary" { +# filename = "${path.module}/resources/portainer-init-snapshot.tar.gz" +# } + + +data "cloudinit_config" "init_docker" { + part { + filename = "cloud-init.yaml" + content_type = "text/cloud-config" + content = templatefile("${path.module}/cloud-init.yaml", + { + PORTAINER_AGENT_SECRET = var.portainer_secrets.agent_secret + } + ) + } +} + +data "cloudinit_config" "init_docker_controller" { + part { + filename = "cloud-init-controller.yaml" + content_type = "text/cloud-config" + content = templatefile("${path.module}/cloud-init-controller.yaml", + { + PORTAINER_AGENT_SECRET = var.portainer_secrets.agent_secret, + PORTAINER_SNAPSHOT_PASSWORD = var.portainer_secrets.snapshot_password + } + ) + } +} + +data "openstack_compute_flavor_v2" "available_compute_flavors" { + for_each = toset(["2cpu4ram", "8cpu16ram"]) + name = each.value +} + + +data "openstack_networking_network_v2" "external_4003" { + name = "external_4003" +} + +data "openstack_images_image_v2" "ubuntu" { + name = var.ubuntu_immage_name + most_recent = true +} + +data "openstack_networking_secgroup_v2" "er_https_from_lbs" { + name = "er_https_from_lbs" +} diff --git a/deployment/terraform/modules/openstack-cogstack-infra/networking.tf b/deployment/terraform/modules/openstack-cogstack-infra/networking.tf new file mode 100644 index 0000000..3651fa0 --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/networking.tf @@ -0,0 +1,46 @@ + + +locals { + devops_controller_cidr = "${local.controller_host_instance.access_ip_v4}/32" + + cogstack_apps_ingress_rules = [ + { port = 22, cidr = var.allowed_ingress_ips_cidr, description = "SSH" }, + { port = 9443, cidr = var.allowed_ingress_ips_cidr, description = "Allow access to Portainer UI to users" }, + { port = 80, cidr = var.allowed_ingress_ips_cidr, description = "Expose traefik to users on port 80" }, + { port = 5000, cidr = var.allowed_ingress_ips_cidr, description = "MedCAT Service API for allowed users" }, + ] + cogstack_apps_devops_controller_rules = [ + { port = 9001, cidr = local.devops_controller_cidr, description = "Allow ingress to portainer agent from the devops controller" }, + { port = 5000, cidr = local.devops_controller_cidr, description = "MedCAT Service API for Probing" }, + ] + cogstack_apps_ingress_rules_map = { for rule in local.cogstack_apps_ingress_rules : "${rule.port}-${rule.description}" => rule } + cogstack_apps_devops_controller_rules_map = { for rule in local.cogstack_apps_devops_controller_rules : "${rule.port}-${rule.description}" => rule } +} + +resource "openstack_networking_secgroup_v2" "cogstack_apps_security_group" { + name = "${local.random_prefix}-cogstack-services" + description = "Cogstack Apps and Services Group" +} + +resource "openstack_networking_secgroup_rule_v2" "cogstack_apps_port_rules" { + for_each = local.cogstack_apps_ingress_rules_map + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = each.value.port + port_range_max = each.value.port + remote_ip_prefix = each.value.cidr + description = each.value.description + security_group_id = openstack_networking_secgroup_v2.cogstack_apps_security_group.id +} +resource "openstack_networking_secgroup_rule_v2" "cogstack_apps_devops_controller_rules" { + for_each = local.cogstack_apps_devops_controller_rules_map + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = each.value.port + port_range_max = each.value.port + remote_ip_prefix = each.value.cidr + description = each.value.description + security_group_id = openstack_networking_secgroup_v2.cogstack_apps_security_group.id +} diff --git a/deployment/terraform/modules/openstack-cogstack-infra/outputs.tf b/deployment/terraform/modules/openstack-cogstack-infra/outputs.tf new file mode 100644 index 0000000..f9a13aa --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/outputs.tf @@ -0,0 +1,28 @@ +output "created_hosts" { + value = { for k, value in openstack_compute_instance_v2.cogstack_ops_compute : k => { + ip_address = value.access_ip_v4 + unique_name = value.name + name = k + } } + + description = "Created Hosts: A map of { hostname: { data } }" +} + +output "created_controller_host" { + value = { + name = (local.controller_host.name) + ip_address = local.controller_host_instance.access_ip_v4 + unique_name = local.controller_host_instance.name + + } + + description = "Created Controller Host: A map of { hostname: { data } }" +} + +output "compute_keypair" { + value = { + public_key_file = local.ssh_keys.public_key_file, + private_key_file = local.ssh_keys.private_key_file, + } + description = "Absolute path to a public and private SSH key pair that is granted login on created VMs" +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-cogstack-infra/providers.tf b/deployment/terraform/modules/openstack-cogstack-infra/providers.tf new file mode 100644 index 0000000..837912b --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/providers.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "~> 3.0.0" + } + } +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-cogstack-infra/shared-locals.tf b/deployment/terraform/modules/openstack-cogstack-infra/shared-locals.tf new file mode 100644 index 0000000..1e9843e --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/shared-locals.tf @@ -0,0 +1,21 @@ + +locals { + random_prefix = random_id.server.b64_url +} + + +locals { + controller_host = one([for host in var.host_instances : host if host.is_controller]) + controller_host_instance = openstack_compute_instance_v2.cogstack_ops_compute[local.controller_host.name] +} + + +resource "random_id" "server" { + keepers = { + # Generate a new id each time we recreate the hosts + cloud_init_config = data.cloudinit_config.init_docker.id + cloud_init_config_controller = data.cloudinit_config.init_docker_controller.id + } + + byte_length = 4 +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-cogstack-infra/variables.tf b/deployment/terraform/modules/openstack-cogstack-infra/variables.tf new file mode 100644 index 0000000..ecc14d4 --- /dev/null +++ b/deployment/terraform/modules/openstack-cogstack-infra/variables.tf @@ -0,0 +1,68 @@ +variable "portainer_secrets" { + type = object({ + agent_secret = string, + snapshot_password = string, + }) + description = <= 1 + error_message = "Must have at least one host instance" + } + + validation { + condition = length([for x in var.host_instances : x if x.is_controller == true]) == 1 + error_message = "Must have exactly one controller host" + } + +} + +variable "ssh_key_pair" { + type = object({ + public_key_file = string + private_key_file = string + }) + default = null + description = "Paths to an SSH Public and Private Keypair. If provided these will be given SSH access on the created hosts. If not provided, a Keypair will be generated and accessible as a local file" + validation { + condition = var.ssh_key_pair == null || fileexists(var.ssh_key_pair.private_key_file) + error_message = "No file exists in SSH private key path" + } + validation { + condition = var.ssh_key_pair == null || fileexists(var.ssh_key_pair.public_key_file) + error_message = "No file exists in SSH public key path" + } +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/README.md b/deployment/terraform/modules/openstack-kubernetes-infra/README.md new file mode 100644 index 0000000..466a0e7 --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/README.md @@ -0,0 +1,38 @@ +# Cogstack Opesntack K3S Module + +Terraform Module for provisioning VMs on Openstack as a kubernetes cluster using K3s + +## Features +- Create a multi node kuberentes cluster in Openstack with K3s +- Create VMs and initialise their setup using K3s install scripts +- Setup Networking in Openstack to allow communications between expected services +- Return a KUBECONFIG file for integration with kubernetes and helm providers in terraform + +## Requirements +- An Openstack Environment able to create Compute and Networking rules +- An accessible Ubuntu image in the openstack environment + +## Example + +```hcl +module "openstack_kubernetes_cluster" { + source = "../../modules/openstack-kubernetes-infra" + host_instances = [ + { name = "cogstack-k3s-server", is_controller = true }, + { + name = "cogstack-k3s-node-2" + flavour = "2cpu4ram" + volume_size = 20 + is_controller = false + }, + ] + allowed_ingress_ips_cidr = "0.0.0.0/24" + ubuntu_immage_name = "Ubuntu_Jammy"" + ssh_key_pair = { + private_key_file = "~/.ssh/id_rsa", + public_key_file = "~/.ssh/id_rsa.pub", + } +} +``` + + diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml new file mode 100644 index 0000000..36aa265 --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml @@ -0,0 +1,45 @@ +#cloud-config +# create the docker group +groups: + - docker + +# Add default auto created user to docker group +system_info: + default_user: + groups: [docker] + +write_files: + - path: /etc/profile.d/k3s-kubeconfig.sh + permissions: '0644' + # Sets up kubectl CLI to work on login + content: | + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +runcmd: +# Install Docker + - echo "Installing Docker" + # Add Docker's official GPG key: + - sudo apt-get update + - sudo apt-get install ca-certificates curl + - sudo install -m 0755 -d /etc/apt/keyrings + - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + - sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + - | + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + - sudo apt-get update + - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + - echo "Completed Installing Docker" + + # Run K3s + - echo "Installing K3S" + - curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" sh - + - echo "Completed Installing K3S" + # - sudo chmod 644 /etc/rancher/k3s/k3s.yaml + + # # Install Helm CLI + # - sudo curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml new file mode 100644 index 0000000..a7c379a --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml @@ -0,0 +1,45 @@ +#cloud-config +# create the docker group +groups: + - docker + +# Add default auto created user to docker group +system_info: + default_user: + groups: [docker] + +write_files: + - path: /etc/profile.d/k3s-kubeconfig.sh + permissions: '0644' + # Sets up kubectl CLI to work on login + content: | + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +runcmd: +# Install Docker + - echo "Installing Docker" + # Add Docker's official GPG key: + - sudo apt-get update + - sudo apt-get install ca-certificates curl + - sudo install -m 0755 -d /etc/apt/keyrings + - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + - sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + - | + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + - sudo apt-get update + - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + - echo "Completed Installing Docker" + + # Run K3s + - echo "Installing K3S" + - sudo curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" K3S_TOKEN="${TF_K3S_TOKEN}" sh - + - echo "Completed Installing K3S" + - sudo chmod 644 /etc/rancher/k3s/k3s.yaml + + # Install Helm CLI + - sudo curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf b/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf new file mode 100644 index 0000000..819bd71 --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf @@ -0,0 +1,33 @@ +# Create a Keypair resource in openstack +# Either generate a brand new public and private key, when the var.ssh_key_pair variable is not provided +# Or use the file contents in var.ssh_key_pair + +locals { + is_using_existing_ssh_keypair = var.ssh_key_pair != null + ssh_keys = { + # If user inputs an existing SSH keypair, then pass it through. Else use the generated resources. + private_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.private_key_file) : openstack_compute_keypair_v2.compute_keypair.private_key + private_key_file = local.is_using_existing_ssh_keypair ? var.ssh_key_pair.private_key_file : abspath(local_file.private_key[0].filename) + public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : openstack_compute_keypair_v2.compute_keypair.public_key + public_key_file = local.is_using_existing_ssh_keypair ? var.ssh_key_pair.public_key_file : abspath(local_file.public_key[0].filename) + } +} + +resource "openstack_compute_keypair_v2" "compute_keypair" { + name = "${local.random_prefix}-cogstack_keypair" + public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : null +} + +resource "local_file" "private_key" { + count = local.is_using_existing_ssh_keypair ? 0 : 1 + content = openstack_compute_keypair_v2.compute_keypair.private_key + filename = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pem" + file_permission = "0600" +} + +resource "local_file" "public_key" { + count = local.is_using_existing_ssh_keypair ? 0 : 1 + content = openstack_compute_keypair_v2.compute_keypair.public_key + filename = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub" + file_permission = "0600" +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf b/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf new file mode 100644 index 0000000..2988872 --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf @@ -0,0 +1,158 @@ + + +resource "openstack_compute_instance_v2" "kubernetes_server" { + + name = "${local.random_prefix}-${local.controller_host.name}" + flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[local.controller_host.flavour].id + key_pair = openstack_compute_keypair_v2.compute_keypair.name + region = "RegionOne" + + user_data = data.cloudinit_config.init_docker_controller.rendered + security_groups = ["default", + openstack_networking_secgroup_v2.cogstack_apps_security_group.name + ] + + network { + uuid = data.openstack_networking_network_v2.external_4003.id + } + + block_device { + uuid = data.openstack_images_image_v2.ubuntu.id + source_type = "image" + volume_size = local.controller_host.volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + connection { + user = "ubuntu" + host = self.access_ip_v4 + private_key = file(local.ssh_keys.private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait > /tmp/openstack-terraform-remote-exec-provisioner.log || true", + ] + } +} + +resource "openstack_compute_instance_v2" "kubernetes_nodes" { + depends_on = [openstack_compute_instance_v2.kubernetes_server] + for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller } + name = "${local.random_prefix}-${each.value.name}" + flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id + key_pair = openstack_compute_keypair_v2.compute_keypair.name + region = "RegionOne" + + user_data = data.cloudinit_config.init_docker.rendered + security_groups = ["default", + openstack_networking_secgroup_v2.cogstack_apps_security_group.name + ] + + network { + uuid = data.openstack_networking_network_v2.external_4003.id + } + + block_device { + uuid = data.openstack_images_image_v2.ubuntu.id + source_type = "image" + volume_size = each.value.volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + connection { + user = "ubuntu" + host = self.access_ip_v4 + private_key = file(local.ssh_keys.private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait > /tmp/openstack-terraform-remote-exec-provisioner.log || true", + ] + } +} + + + +# TODO: Read content from files and put into cloud-init config +# data "local_file" "install_docker_sh" { +# filename = "${path.module}/resources/install-docker.sh" +# } + +# data "local_file" "initialize_portainer_binary" { +# filename = "${path.module}/resources/portainer-init-snapshot.tar.gz" +# } + + +data "cloudinit_config" "init_docker" { + depends_on = [openstack_compute_instance_v2.kubernetes_server] + part { + filename = "cloud-init-k3s-agent.yaml" + content_type = "text/cloud-config" + content = templatefile("${path.module}/cloud-init-k3s-agent.yaml", + { + TF_K3S_TOKEN = random_password.k3s_token.result + TF_K3S_SERVER_IP_ADDRESS = openstack_compute_instance_v2.kubernetes_server.access_ip_v4 + } + ) + } +} + +resource "random_password" "k3s_token" { + length = 32 +} + +data "cloudinit_config" "init_docker_controller" { + part { + filename = "cloud-init-k3s-server.yaml" + content_type = "text/cloud-config" + content = templatefile("${path.module}/cloud-init-k3s-server.yaml", + { + TF_K3S_TOKEN = random_password.k3s_token.result + } + ) + } +} + +data "openstack_compute_flavor_v2" "available_compute_flavors" { + for_each = toset(["2cpu4ram", "8cpu16ram"]) + name = each.value +} + + +data "openstack_networking_network_v2" "external_4003" { + name = "external_4003" +} + +data "openstack_images_image_v2" "ubuntu" { + name = var.ubuntu_immage_name + most_recent = true +} + +data "openstack_networking_secgroup_v2" "er_https_from_lbs" { + name = "er_https_from_lbs" +} + +resource "null_resource" "copy_kubeconfig" { + depends_on = [openstack_compute_instance_v2.kubernetes_server] + + provisioner "local-exec" { + # Copy the kubeconfig file from the host to a local file using SCP. + # Use ssh-keyscan to prevent interactive prompt on unknown host + # Use sed to replace the localhost address in the KUBECONFIG file with the actual IP adddress of the created VM. + command = <> ${path.module}/.build/.known_hosts_cogstack && \ +scp -o UserKnownHostsFile=${path.module}/.build/.known_hosts_cogstack -o StrictHostKeyChecking=yes \ + -i ${local.ssh_keys.private_key_file} \ + ubuntu@${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}:/etc/rancher/k3s/k3s.yaml \ + ${local.kubeconfig_file} && \ +sed -i "s/127.0.0.1/${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}/" ${local.kubeconfig_file} +EOT + } +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf b/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf new file mode 100644 index 0000000..a646c9d --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf @@ -0,0 +1,31 @@ + + +locals { + devops_controller_cidr = "${local.controller_host_instance.access_ip_v4}/32" + + cogstack_apps_ingress_rules = [ + { port = 22, cidr = var.allowed_ingress_ips_cidr, description = "Expose SSH" }, + { port = 80, cidr = var.allowed_ingress_ips_cidr, description = "Expose port 80 (hhtp) for ingress" }, + { port = 443, cidr = var.allowed_ingress_ips_cidr, description = "Expose port 443 (https) for ingress" }, + { port = 6443, cidr = var.allowed_ingress_ips_cidr, description = "Expose kubernetes for CLI" } + ] + cogstack_apps_ingress_rules_map = { for rule in local.cogstack_apps_ingress_rules : "${rule.port}-${rule.description}" => rule } +} + +resource "openstack_networking_secgroup_v2" "cogstack_apps_security_group" { + name = "${local.random_prefix}-cogstack-services" + description = "Cogstack Apps and Services Group" +} + +resource "openstack_networking_secgroup_rule_v2" "cogstack_apps_port_rules" { + for_each = local.cogstack_apps_ingress_rules_map + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = each.value.port + port_range_max = each.value.port + remote_ip_prefix = each.value.cidr + description = each.value.description + security_group_id = openstack_networking_secgroup_v2.cogstack_apps_security_group.id +} + diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf b/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf new file mode 100644 index 0000000..3cf1097 --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf @@ -0,0 +1,33 @@ + +output "created_hosts_2" { + value = { for k, value in openstack_compute_instance_v2.kubernetes_nodes : k => { + ip_address = value.access_ip_v4 + unique_name = value.name + name = k + } } + + description = "Created Hosts: A map of { hostname: { data } }" +} + +output "created_controller_host" { + value = { + name = (local.controller_host.name) + ip_address = local.controller_host_instance.access_ip_v4 + unique_name = local.controller_host_instance.name + } + + description = "Created Controller Host: A map of { hostname: { data } }" +} + +output "compute_keypair" { + value = { + public_key_file = local.ssh_keys.public_key_file, + private_key_file = local.ssh_keys.private_key_file, + } + description = "Absolute path to a public and private SSH key pair that is granted login on created VMs" +} + +output "kubeconfig_file" { + value = abspath(local.kubeconfig_file) + description = "Path to the generated KUBECONFIG file used to connect to kubernetes" +} diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/providers.tf b/deployment/terraform/modules/openstack-kubernetes-infra/providers.tf new file mode 100644 index 0000000..837912b --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/providers.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "~> 3.0.0" + } + } +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf b/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf new file mode 100644 index 0000000..53b6458 --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf @@ -0,0 +1,23 @@ + +locals { + random_prefix = random_id.server.b64_url +} + + +locals { + controller_host = one([for host in var.host_instances : host if host.is_controller]) + controller_host_instance = openstack_compute_instance_v2.kubernetes_server +} + +locals { + kubeconfig_file = "${path.module}/.build/downloaded-kubeconfig.yaml" +} + +resource "random_id" "server" { + keepers = { + # Generate a new id each time we recreate the hosts + cloud_init_config_controller = data.cloudinit_config.init_docker_controller.id + } + + byte_length = 4 +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf b/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf new file mode 100644 index 0000000..90732fe --- /dev/null +++ b/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf @@ -0,0 +1,57 @@ +variable "allowed_ingress_ips_cidr" { + type = string + default = "0.0.0.0/24" + description = "CIDR Block that is allowed ingress to deployed services" +} + +variable "ubuntu_immage_name" { + type = string + description = "Name of an available Machine Image running ubuntu in the openstack environment" +} + +variable "host_instances" { + description = <= 1 + error_message = "Must have at least one host instance" + } + + validation { + condition = length([for x in var.host_instances : x if x.is_controller == true]) == 1 + error_message = "Must have exactly one controller host" + } + +} + +variable "ssh_key_pair" { + type = object({ + public_key_file = string + private_key_file = string + }) + default = null + description = "Paths to an SSH Public and Private Keypair. If provided these will be given SSH access on the created hosts. If not provided, a Keypair will be generated and accessible as a local file" + validation { + condition = var.ssh_key_pair == null || fileexists(var.ssh_key_pair.private_key_file) + error_message = "No file exists in SSH private key path" + } + validation { + condition = var.ssh_key_pair == null || fileexists(var.ssh_key_pair.public_key_file) + error_message = "No file exists in SSH public key path" + } +} \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 7df8c83..f1969be 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,7 +37,7 @@ MedCAT Trainer :caption: CogStack Platform NiFi - +platform/deployment/_index.md platform/_index ``` diff --git a/docs/overview/CogStack ecosystem (v1).md b/docs/overview/CogStack ecosystem (v1).md index 36fab95..2913be5 100644 --- a/docs/overview/CogStack ecosystem (v1).md +++ b/docs/overview/CogStack ecosystem (v1).md @@ -35,7 +35,7 @@ Usually, the sink will be the ElasticSearch store, keeping the processed EHRs wh The information about available data processing components offered by CogStack Pipeline can be found in [CogStack Pipeline](CogStack%20Pipeline.md) part. -:::{ifno} +:::{info} We recommend using CogStack Pipeline component in the newest version 1.3.0. ::: diff --git a/docs/platform/deployment/_index.md b/docs/platform/deployment/_index.md new file mode 100644 index 0000000..91a51cd --- /dev/null +++ b/docs/platform/deployment/_index.md @@ -0,0 +1,42 @@ +# Deployment + +## Introduction +:::{warning} +We are actively working on improving the deployment of CogStack + +The following section shows how to run MedCAT Service in a variety of environments and configs. + +To setup a full deployment of CogStack including the data pipelines and infrastructure, contact us +::: + +## Overview +CogStack is a self-hosted platform made up of microservices and tools + +If you want to get started quickly, check out the [Quickstart](./get-started/quickstart) guide to run CogStack locally in just a few simple steps. + +Our recommended deployment method is on Kubernetes using Helm charts. This makes installing and managing CogStack easy and consistent. For a detailed walkthrough, see [Helm](./helm/_index) . + +You can run Kubernetes anywhere — on your own hardware or through cloud providers like AWS (EKS) or Azure (AKS). To help with this, we provide basic examples using Terraform that will deploy the infrastructure, services, and perform tests using a few terraform commands. These are available in the [Examples](./examples/_index) folder. + +Along with kubernetes, it is also possible to run CogStack through docker compose. See the [Reference](./reference/_index) folder for this. + +## Deployment Recommendations + +- Deploy CogStack on Kubernetes for best scalability and reliability. +- Use Helm to install and manage your CogStack instances. +- Manage your infrastructure declaratively with Terraform. +- Keep your Terraform code in your own Git repository and adopt GitOps practices + + +## Contents +```{toctree} +:maxdepth: 2 + +get-started/_index + +helm/_index + +examples/_index + +reference/_index +``` diff --git a/docs/platform/deployment/examples/_index.md b/docs/platform/deployment/examples/_index.md new file mode 100644 index 0000000..ed633c6 --- /dev/null +++ b/docs/platform/deployment/examples/_index.md @@ -0,0 +1,17 @@ +# Examples + +These examples demonstrate how to deploy CogStack infrastructure quickly using Terraform. Each example typically requires just one or two `terraform apply` commands to create all the required infrastructure and run CogStack. + +Use them as a starting point or reference for your own deployments. + +:::{warn} +Please note these examples are for demonstration and testing only. We do not advise using these with any real production data without review. Cloud infrastructure can be complex, and your IT department likely has guidelines or governance rules you’ll need to follow when moving beyond these basic setups. +::: + +```{toctree} +:maxdepth: 2 +aws-kubernetes-eks +azure-kubernetes-aks +openstack-kubernetes-k3s +openstack-docker +``` diff --git a/docs/platform/deployment/examples/aws-kubernetes-eks.md b/docs/platform/deployment/examples/aws-kubernetes-eks.md new file mode 100644 index 0000000..ae17bc3 --- /dev/null +++ b/docs/platform/deployment/examples/aws-kubernetes-eks.md @@ -0,0 +1,101 @@ +# AWS EKS Deployment + +This is an example deployment of CogStack in AWS. It will create publically accessible services, so is not suitable for production deployment. + +The recommended deployment in AWS is based on using Kubernetes through AWS EKS. + +This example will create a AWS EKS cluster, setup any necessary config, deploy CogStack to the cluster, and test that it is available. + +## Usage +Deployment through terraform is carried out through two terraform commands, to handle the sequencing issues between making a k8s cluster and using it in AWS. + +### Requirements +- Terraform - [Install Terraform](https://developer.hashicorp.com/terraform/install) +- AWS Credentials for an account that can create and destroy resources. + + +### Steps + +### 1. Add Required Secrets for your env +This readme uses environment variables for access: + +1. See the `.env.example` file for the required details. +2. Create a file `.env` with those fields set for your account. +3. Execute `source .env` to set those environment variables + +If desired, see the official documentation for other ways to provide AWS credentials https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration + +### 2. Run Terraform +Terraform is run on two modules for AWS, so we will run one terraform apply in one folder, then another terraform apply in a second folder. + +Initial provisioning takes around 15 minutes. + +```bash +# Set AWS credentials +source .env + +# Create AWS EKS infra +cd eks-cluster +terraform init +terraform apply --auto-approve + +AWS_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# Deploy services to kubernetes +cd ../kubernetes-deployment +export TF_VAR_kubeconfig_file=$AWS_KUBECONFIG +terraform init +terraform apply --auto-approve +``` + +### 3. Accessing the CogStack Platform + +Once the deployment is complete and all services are running, you can access the CogStack platform and its components using the following URLs: + +```bash +terraform output service_urls +``` + + +### Optional - Destroy + +You can destroy the infra to save costs when it wont be used for a long time. + +Do note that there is an initial cost every time the EKS infrastructure is created, looks to be around $0.50 at time of writing. + +```bash +cd ../kubernetes-deployment +terraform destroy + +cd ../eks-cluster +terraform destroy +``` + + +## Optionally use the K8s cluster as normal with the CLI +After setting up the cluster, it is possible to interact directly with it using the kubectl CLI + +The requirement is to get the KUBECONFIG file created by the terraform apply. + +```bash +# Get KUBECONFIG +cd eks-cluster +AWS_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# SET KUBECONFIG +export KUBECONFIG=${AWS_KUBECONFIG} +``` + +Note - alternatively you could use the AWS CLI to set your kubeconfig using `aws eks update-kubeconfig --name $(terraform output -raw cluster_name)`. + +You can then interact with kubernetes via the CLI + +```bash +# Run Medcat service +helm install my-medcat oci://registry-1.docker.io/cogstacksystems/medcat-service-helm --wait --timeout 10m0s + +# Create the ingress +kubectl apply -f resources/ingress-medcat-service.yaml +# Find public url +kubectl get ingress +``` \ No newline at end of file diff --git a/docs/platform/deployment/examples/azure-kubernetes-aks.md b/docs/platform/deployment/examples/azure-kubernetes-aks.md new file mode 100644 index 0000000..83a2212 --- /dev/null +++ b/docs/platform/deployment/examples/azure-kubernetes-aks.md @@ -0,0 +1,115 @@ +# Azure AKS Deployment + +This is an example deployment of CogStack in Azure. + +The recommended deployment of CogStack in Azure is based on using Kubernetes through Azure Kubernetes Service. + +This example will create a AKS cluster, setup any necessary config, deploy CogStack to the cluster, and test that it is available. It will create publically accessible services, so is not suitable for production deployment. + +We create a cluster following the Official Azure Verified Modules patterns in https://azure.github.io/Azure-Verified-Modules/indexes/terraform/tf-pattern-modules/ to create AKS clusters with their recommended defaults. + + +## Usage +Deployment through terraform is carried out through two terraform commands, to handle the sequencing issues between making a k8s cluster and using it in the cloud. + +### Requirements +- Terraform - [Install Terraform](https://developer.hashicorp.com/terraform/install) +- Azure Credentials for an account and subscription that can create and destroy resources. + +#### Required Permissions +- Contributor +- User Access Administrator +... +#### Required Features +- EncryptionAtHost: `az feature register --namespace Microsoft.Compute --name EncryptionAtHost` + +### Steps + +### 1. Use the Azure CLI to login for your subscription +Run the az login command, which will open a web browser for you to login to your azure account. We then set the subscription ID for use by the Azure RM Terraform provider. + +```bash +az login +export ARM_SUBSCRIPTION_ID=$(az account show --query id -o tsv) +``` + +### 2. Run Terraform +Terraform is run on two modules, so we will run one terraform apply in one folder, then another terraform apply in a second folder. + +Initial provisioning takes around 15 minutes. + +```bash +# Create AKS cluster +cd aks-cluster +terraform init +terraform apply --auto-approve + +AZURE_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# Deploy services to kubernetes +cd ../kubernetes-deployment +export TF_VAR_kubeconfig_file=$AZURE_KUBECONFIG +terraform init +terraform apply --auto-approve +``` + +### 3. Accessing the CogStack Platform + +Once the deployment is complete and all services are running, you can access the CogStack platform and its components using the following URLs: + +TODO: Create a public ingress url +```bash +# terraform output service_urls +kubectl port-forward deployment/medcat-service-terraform-medcat-service-helm 5000:5000 +http://localhost:5000/demo +``` + + +### Optional - Destroy + +You can destroy the infra to save costs when it wont be used for a long time. + +Do note that there is an initial cost every time the EKS infrastructure is created, looks to be around $0.50 at time of writing. + +```bash +cd ../kubernetes-deployment +terraform destroy + +cd ../aks-cluster +terraform destroy +``` + + +## Optionally use the K8s cluster as normal with the CLI +After setting up the cluster, it is possible to interact directly with it using the kubectl CLI + +The requirement is to get the KUBECONFIG file created by the terraform apply. + +```bash +# Get KUBECONFIG +cd aks-cluster +AZURE_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# SET KUBECONFIG +export KUBECONFIG=${AZURE_KUBECONFIG} +``` + +Note - alternatively you could use the Azure CLI to set your kubeconfig using + +```bash +MY_RESOURCE_GROUP_NAME=$(terraform output -raw resource_group_name) +MY_AKS_CLUSTER_NAME=$(terraform output -raw cluster_name) +az aks get-credentials --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_AKS_CLUSTER_NAME` +``` + +You can then interact with kubernetes via the CLI for example: + +```bash +# Run Medcat service +helm install my-medcat oci://registry-1.docker.io/cogstacksystems/medcat-service-helm --wait --timeout 10m0s + +# Create the ingress +kubectl apply -f resources/ingress-medcat-service.yaml +# Find public url +kubectl get ingress +``` \ No newline at end of file diff --git a/docs/platform/deployment/examples/openstack-docker.md b/docs/platform/deployment/examples/openstack-docker.md new file mode 100644 index 0000000..090c545 --- /dev/null +++ b/docs/platform/deployment/examples/openstack-docker.md @@ -0,0 +1,62 @@ +# Openstack Docker Deployment + +This Terraform example provides one stop approach to deploy the **CogStack** platform with its core components and observability stack in an OpenStack environment. It is specifically designed to simplify and automate the provisioning and configuration needed to run CogStack reliably and securely. + +This example: + +- **Provisions Ubuntu VMs** in openstack +- **Installs Docker and Portainer** on the VMs using Cloud-Init to manage containers easily +- **Uses Ansible for Configuration Management** to deploy necessary CogStack configuration files and system setup +- **Deploys the MedCAT Service** enabling natural language processing on an API +- **Sets up Observability Tools** by deploying Prometheus for metrics collection and Grafana dashboards for monitoring the health and performance of CogStack services. +- **Runs Integration Tests** after the infrastructure is created, asserting that services are running on the created IP addresses. + +## Usage + +### Requirements +- Terraform - [Install Terraform](https://developer.hashicorp.com/terraform/install) +- Openstack Cloud environment + +### 1. Add Required Secrets for your env + +Create a `terraform.tfvars` file, based on `terraform.tfvars.example`, containing the secrets for your environment. + +### 2. Run Terraform + +```bash +terraform init +terraform apply +``` + +Initial provisioning takes up to 10 minutes, where time is mostly downloading large docker images + +### 3. Accessing the CogStack Platform + +Once the deployment is complete and all services are running, you can access the CogStack platform and its components using the following URLs: + +```bash +terraform output service_urls +``` + +## Troubleshooting + + +### unsupported protocol scheme +If you make changes to the created VM infrastructure, and want to reapply, you can run into this error + +``` +│ Error: Get "/api/endpoints/4": unsupported protocol scheme "" +│ +│ with module.cogstack_docker_services.portainer_environment.portainer_envs["cogstack-devops"], +│ on ../../modules/cogstack-docker-services/environments.tf line 3, in resource "portainer_environment" "portainer_envs": +│ 3: resource "portainer_environment" "portainer_envs" { +``` + +Fix by targetting just the infra module first: + +```bash +terraform apply -target=module.openstack_cogstack_infra +terraform apply +``` + +For details: the error specifically occurs after making a change to the controller host, forcing it to be deleted and recreated, however terraform still uses the IP address in the portainer provider. Targetting just the infra module first, means terraform wont call any APIs during the plan stage using the old IP address. diff --git a/docs/platform/deployment/examples/openstack-kubernetes-k3s.md b/docs/platform/deployment/examples/openstack-kubernetes-k3s.md new file mode 100644 index 0000000..311a539 --- /dev/null +++ b/docs/platform/deployment/examples/openstack-kubernetes-k3s.md @@ -0,0 +1,75 @@ +# Openstack Kubernetes Deployment + +This Terraform example provides one stop approach to deploy the **CogStack** platform with its core components and observability stack in an OpenStack environment. It is specifically designed to simplify and automate the provisioning and configuration needed to run CogStack reliably and securely. + +This example: + +- **Provisions Ubuntu VMs** in openstack +- **Installs Docker** on the VMs using Cloud-Init to manage containers easily +- **Installs Kubernetes using k3s** using Cloud-init + +## Usage + +### Requirements +- Terraform - [Install Terraform](https://developer.hashicorp.com/terraform/install) +- Openstack Cloud environment + +### 1. Add Required Secrets for your env + +Create a `terraform.tfvars` file, based on `terraform.tfvars.example`, containing the secrets for your environment. + +### 2. Run Terraform + +```bash +# Create AKS cluster +cd k3s-cluster +terraform init +terraform apply --auto-approve + +K3S_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# Deploy services to kubernetes +cd ../kubernetes-deployment +export TF_VAR_kubeconfig_file=$K3S_KUBECONFIG +terraform init +terraform apply --auto-approve +``` + +Initial provisioning takes up to 10 minutes, where time is mostly downloading large docker images + +### 3. Accessing the CogStack Platform + +Once the deployment is complete and all services are running, you can access the CogStack platform and its components using the following URLs: + +```bash +terraform output created_services +``` + +## Optionally use the K8s cluster as normal with the CLI +After setting up the cluster, it is possible to interact directly with it using the kubectl CLI + +The requirement is to get the KUBECONFIG file created by the terraform apply. + +```bash +# Get KUBECONFIG +K3S_KUBECONFIG=$(terraform output -raw kubeconfig_file) + +# SET KUBECONFIG +export KUBECONFIG=${K3S_KUBECONFIG} +``` + +You can then interact with kubernetes via the CLI for example: + +```bash +# Run Medcat service +helm install my-medcat oci://registry-1.docker.io/cogstacksystems/medcat-service-helm --wait --timeout 10m0s + +# Find public url +kubectl get ingress +``` + +Access the k8s dashboard using +``` +terraform output dashboard # Find the access token +kubectl -n kubernetes-dashboard port-forward svc/kubernetes-dashboard-kong-proxy 8443:443 +``` \ No newline at end of file diff --git a/docs/platform/deployment/get-started/_index.md b/docs/platform/deployment/get-started/_index.md new file mode 100644 index 0000000..f6fad99 --- /dev/null +++ b/docs/platform/deployment/get-started/_index.md @@ -0,0 +1,36 @@ +# Getting Started + +This page outlines the basic setup and requirements for running CogStack + +See the Quickstart guide for a tutorial that will install a local kubernetes using Minikube, and run an instance of CogStack using Helm. + +## Models +The primary thing you will need to arrange is a trained model to be used for the natural language processing functionality of CogStack. We provide small free models, though to get access to better performing models please contact us + +## Technologies and Tools +Our recommended deployment method is on **Kubernetes** by using **Helm** charts + +These are some of the terms and technologies relevant to deploying CogStack: + +- [GitOps](https://en.wikipedia.org/wiki/DevOps#GitOps) +- [Docker](https://docs.docker.com/get-docker/) +- [Terraform](https://www.terraform.io/downloads) +- [Kubernetes](https://kubernetes.io/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/) +- [Helm](https://helm.sh/docs/intro/install/) +- [Ansible](https://docs.ansible.com/ansible/latest/index.html) +- [Portainer](https://www.portainer.io/) + +See the official documentation on these tools for the best documentation for installation and setup. Not all of these are needed depending on which deployment method is used. + +## Cloud Accounts & Permissions + +If you plan to deploy on cloud providers like AWS, Azure, or OpenStack, ensure you have the appropriate accounts set up with necessary permissions. Refer to the respective provider’s documentation for guidance. + + +## Contents +```{toctree} +:maxdepth: 2 + +quickstart +``` diff --git a/docs/platform/deployment/get-started/quickstart.md b/docs/platform/deployment/get-started/quickstart.md new file mode 100644 index 0000000..d0d605c --- /dev/null +++ b/docs/platform/deployment/get-started/quickstart.md @@ -0,0 +1,64 @@ + +# Quickstart + +A local setup guide for running k8s locally, and installing with helm + + +## Requirements +- Docker installed ([install Docker](https://docs.docker.com/get-docker/)) +- Docker Compose installed ([install Docker Compose](https://docs.docker.com/compose/install/)) +- A terminal with network access + +## Setup + +1. **Install Minikube** + +You can install Minikube by following the official instructions: +https://minikube.sigs.k8s.io/docs/start/ + +Or just run this if on linux: + +```bash +curl -LO https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-amd64 +sudo install minikube-linux-amd64 /usr/local/bin/minikube && rm minikube-linux-amd64 +``` + +2. **Install Helm** + +Run: + +```bash +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +``` + +3. **Start Minikube** + +```bash +minikube start +``` + +This will create a local Kubernetes cluster. + +## Deploy CogStack with Helm + +Run this command to install the MedCAT service Helm chart: + +```bash +helm install my-medcat oci://registry-1.docker.io/cogstacksystems/medcat-service-helm --wait --timeout 10m0s +``` + +The `--wait` flag makes Helm wait until all resources are ready, and `--timeout` sets a maximum wait time. + +### Verify deployment + +Check that the pods are running: + +```bash +helm test medcat-service --logs +``` + +You’re now running CogStack locally on Kubernetes! For more detailed usage, see the [Helm tutorial](../helm/tutorial.md). + +### Access the service + + diff --git a/docs/platform/deployment/helm/_index.md b/docs/platform/deployment/helm/_index.md new file mode 100644 index 0000000..ce83ae8 --- /dev/null +++ b/docs/platform/deployment/helm/_index.md @@ -0,0 +1,20 @@ +# Helm + + +Helm is our recommended way to deploy CogStack. It simplifies installing, upgrading, and managing all the components through easy-to-use charts. + +For a detailed, step-by-step walkthrough on deploying CogStack with Helm, please see the [Tutorial](./tutorial.md). + +To understand how the Helm charts are structured and how they work under the hood, check out the [architecture](./architecture) section. + + +## Contents + +```{toctree} +:maxdepth: 2 + +tutorial +architecture +cogstack-helm-module + +``` diff --git a/docs/platform/deployment/helm/architecture.md b/docs/platform/deployment/helm/architecture.md new file mode 100644 index 0000000..41f0e91 --- /dev/null +++ b/docs/platform/deployment/helm/architecture.md @@ -0,0 +1,14 @@ +# Architecture + +## Charts + +The Helm charts for CogStack are published to Docker Hub, which is an OCI-compliant registry. + +### Chart Listings + +- **MedCAT Service:** + https://hub.docker.com/r/cogstacksystems/medcat-service-helm + +### Chart Publishing + +Charts are published automatically via a GitHub Action on every commit to the main branch. \ No newline at end of file diff --git a/docs/platform/deployment/helm/cogstack-helm-module.md b/docs/platform/deployment/helm/cogstack-helm-module.md new file mode 100644 index 0000000..7e82684 --- /dev/null +++ b/docs/platform/deployment/helm/cogstack-helm-module.md @@ -0,0 +1,28 @@ + +# CogStack Helm Terraform Module +This Terraform module deploys CogStack services using Helm charts on a Kubernetes cluster. + +## Example usage + +```hcl +module "cogstack_helm_services" { + source = "path_to_module" + medcat_service_values = <