diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..cd02e8a9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,83 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Virtual environment +.venv/ +venv/ +env/ + +# IDE and Editor files +.vscode/ +.idea/ +*.swp +*.bak +*.swo + +# Gitignore and other version control files +.git/ +.gitignore +.gitmodules + +# Environment variables +.env +.env.* +*.env + +# Logs and temporary files +*.log +tmp/ +temp/ + +# Database files +*.sqlite3 +*.db + +# Testing +.cache +.coverage +.tox +coverage.xml +htmlcov +pep8.txt +scratch +testem.log +.pytest_cache/ + +# Mac OS X +*.DS_Store + +# VSCode +.vscode/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 817c064e..aa2d87cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest env: DJANGO_SETTINGS_MODULE: pattern_service.settings + COMPOSE_COMMAND: docker compose steps: - name: Checkout code @@ -32,5 +33,8 @@ jobs: python -m pip install --upgrade pip python -m pip install tox + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + - name: Run unit tests - run: tox -e test + run: make test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a14d4654..4ef8a4e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,9 +11,11 @@ Hi there! We're excited to have you as a contributor. - [Clone the repo](#clone-the-repo) - [Configure Python environment](#configure-python-environment) - [Set env variables for development](#set-env-variables-for-development) + - [Configure postgres and run the dispatcher service](#configure-postgres-and-run-the-dispatcher-service) - [Configure and run the application](#configure-and-run-the-application) - [Updating dependencies](#updating-dependencies) - - [Running tests, linters, and code checks](#running-tests-linters-and-code-checks) + - [Running linters and code checks](#running-linters-and-code-checks) + - [Running tests](#running-tests) ## Things to know prior to submitting code @@ -21,7 +23,6 @@ Hi there! We're excited to have you as a contributor. - Take care to make sure no merge commits are in the submission, and use `git rebase` vs `git merge` for this reason. - If collaborating with someone else on the same branch, consider using `--force-with-lease` instead of `--force`. This will prevent you from accidentally overwriting commits pushed by someone else. For more information, see [git push docs](https://git-scm.com/docs/git-push#git-push---force-with-leaseltrefnamegt). - We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com) -- This repository uses a`pre-commit`, configuration, so ensure that you install pre-commit globally for your user, or by using pipx. ## Build and Run the Development Environment @@ -47,6 +48,8 @@ Install required python modules for development `pip install -r requirements/requirements-dev.txt` +For standalone development tools written in Python, such as `pre-commit` and `pip-tools`, we recommend using your system package manager, `pipx` tool or `pip` user install mode (`pip install --user`), in decreasing order of preference. + ### Set env variables for development Either create a .env file in the project root containing the following env variables, or export them to your shell env: @@ -55,8 +58,29 @@ Either create a .env file in the project root containing the following env varia PATTERN_SERVICE_MODE=development ``` +### Configure postgres and run the dispatcher service + +Several endpoints in the pattern service rely on asynchronous tasks that are handled by a separate running service, the dispatcher service. This uses [PostgreSQL's](https://www.postgresql.org/) `pg_notify` ability to send asyncronous tasks from the django application to the dispatcher service. For more details, see the [dispatcherd documentation](https://github.com/ansible/dispatcherd/blob/main/README.md). + +To make use of the dispatcher, you will need to ensure that both postgres and the dispatcher service are running. _The easiest way to do this is via [docker-compose](./tools/container/README.md)_, however it is also possible to do this manually as follows: + +- Install postgres locally and create a database for the service. +- Update your local .env file to reference your postgres server and database details (these can also be exported to your shell env): + +```bash +PATTERN_SERVICE_DB_NAME= +PATTERN_SERVICE_DB_USER= +PATTERN_SERVICE_DB_PASSWORD= +PATTERN_SERVICE_DB_HOST=localhost +PATTERN_SERVICE_DB_PORT="5432 (or your postgres port)" +``` + +- Run the dispatcherd service from the root pattern service directory with `python manage.py worker` + ### Configure and run the application +In a separate terminal window, run: + `python manage.py migrate && python manage.py runserver` The application can be reached in your browser at `https://localhost:8000/`. The Django admin UI is accessible at `https://localhost:8000/admin` and the available API endpoints will be listed in the 404 information at `http://localhost:8000/api/pattern-service/v1/`. @@ -67,13 +91,22 @@ Project dependencies for all environments are specified in the [pyproject.toml f To add a new dependency: -1. Add the package to the appropriate project or optional dependencies section of the pyproject.toml file, using dependency specifiers to constrain versions. -2. Update the requirements files with the command `make requirements`. This should update the relevant requirements.txt files in the project's requirements directory. +1. Ensure you have `pip-tools` installed by running either `pipx install pip-tools` or `pip install -u pip-tools`. +2. Add the package to the appropriate project or optional dependencies section of the pyproject.toml file, using dependency specifiers to constrain versions. +3. Update the requirements files with the command `make requirements`. This should update the relevant requirements.txt files in the project's requirements directory. + +## Running linters and code checks + +Linters, type checks, and other checks can all be run via `tox`. To see the available `tox` commands for this project, run `tox list`. -## Running tests, linters, and code checks +To run an individual tox command use the `-e` flag to specify the environment, for example: `tox -e lint` to run the linters. -Unit tests, linters, type checks, and other checks can all be run via `tox`. To see the available `tox` commands for this project, run `tox list`. +To run all checks, simply run `tox` with no options. -To run an individual tox command use the `-e` flag to specify the environment, for example: `tox -e test` to run tests with all supported python versions. -s -To run all tests and checks, simply run `tox` with no options. +## Running tests + +Running the tests requires a postgres connection. The easiest way to do this is with the [test compose file](./tools/podman/compose-test.yaml), and there is a `make` command to simplify starting the postgres container and running the tests: + +```bash +make test +``` diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 3453f514..00000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,21 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi:latest - -WORKDIR /app - -COPY requirements/requirements.txt . - -RUN dnf install python3-pip -y - -RUN python3 -m pip install --no-cache-dir -r requirements.txt - -ADD core /app/core - -ADD pattern_service /app/pattern_service - -COPY manage.py . - -RUN python3 manage.py migrate - -EXPOSE 5000 - -CMD ["python3", "/app/manage.py", "runserver", "0.0.0.0:5000"] diff --git a/Makefile b/Makefile index 5f21cd8f..83160a72 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,11 @@ help: ## Show this help message # ------------------------------------- CONTAINER_RUNTIME ?= podman +COMPOSE_COMMAND ?= podman compose IMAGE_NAME ?= pattern-service IMAGE_TAG ?= latest +QUAY_NAMESPACE ?= ansible +BUILD_ARGS ?= --arch amd64 ensure-namespace: @test -n "$$QUAY_NAMESPACE" || (echo "Error: QUAY_NAMESPACE is required to push quay.io" && exit 1) @@ -21,7 +24,7 @@ ensure-namespace: .PHONY: build build: ## Build the container image @echo "Building container image..." - $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f Dockerfile.dev --arch amd64 . + $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f tools/podman/Containerfile.dev $(BUILD_ARGS) . .PHONY: clean clean: ## Remove container image @@ -34,6 +37,24 @@ push: ensure-namespace build ## Tag and push container image to Quay.io $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):$(IMAGE_TAG) quay.io/$(QUAY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG) $(CONTAINER_RUNTIME) push quay.io/$(QUAY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG) +#-------------------------------------- +# Compose +# ------------------------------------- +.PHONY: compose-build +compose-build: ## Build the containers images for the services + $(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) build + +.PHONY: compose-up ## Build and start the containers for the services +compose-up: + $(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans + +.PHONY: compose-down +compose-down: ## Stop containers and remove containers, network, images and volumes created by compose-up + $(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) down --remove-orphans + +.PHONY: compose-restart +compose-restart: compose-down compose-up ## Stop and remove existing infrastructure and start a new one + # ------------------------------------- # Dependencies # ------------------------------------- @@ -43,3 +64,13 @@ requirements: ## Generate requirements.txt files from pyproject.toml pip-compile -o requirements/requirements.txt pyproject.toml pip-compile --extra dev --extra test -o requirements/requirements-dev.txt pyproject.toml pip-compile --extra test -o requirements/requirements-test.txt pyproject.toml + +# ------------------------------------- +# Test +# ------------------------------------- + +.PHONY: test +test: ## Run tests with a postgres database using docker-compose + $(COMPOSE_COMMAND) -f tools/podman/compose-test.yaml $(COMPOSE_OPTS) up -d + -tox -e test + $(COMPOSE_COMMAND) -f tools/podman/compose-test.yaml $(COMPOSE_OPTS) down diff --git a/core/apps.py b/core/apps.py index c0ce093b..169764e9 100644 --- a/core/apps.py +++ b/core/apps.py @@ -1,6 +1,16 @@ +import logging + +from dispatcherd.config import setup as dispatcher_setup from django.apps import AppConfig +from django.conf import settings + +logger = logging.getLogger(__name__) class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "core" + + def ready(self) -> None: + # Configure dispatcher + dispatcher_setup(config=settings.DISPATCHER_CONFIG) diff --git a/core/management/__init__.py b/core/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/management/commands/__init__.py b/core/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/management/commands/worker.py b/core/management/commands/worker.py new file mode 100644 index 00000000..72763de5 --- /dev/null +++ b/core/management/commands/worker.py @@ -0,0 +1,14 @@ +import logging + +from dispatcherd import run_service +from django.core.management.base import BaseCommand + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """Wrapper for worker command.""" + + def handle(self, *args: tuple, **options: dict) -> None: + logger.info("Starting Pattern service dispatcherd worker.") + run_service() diff --git a/core/tasks/__init__.py b/core/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/tasks/demo.py b/core/tasks/demo.py new file mode 100644 index 00000000..338ba0f8 --- /dev/null +++ b/core/tasks/demo.py @@ -0,0 +1,18 @@ +from dispatcherd.publish import submit_task +from dispatcherd.publish import task + +DISPATCHERD_DEFAULT_CHANNEL = "pattern-service-tasks" + + +@task(queue=DISPATCHERD_DEFAULT_CHANNEL, decorate=False) +def print_text(text: str) -> None: + print(text) + + +def sumbit_hello_world(text: str): # type: ignore + job_data, queue = submit_task( + print_text, + queue=DISPATCHERD_DEFAULT_CHANNEL, + args=(text,), + ) + return job_data["uuid"] diff --git a/core/tasks/hazmat.py b/core/tasks/hazmat.py new file mode 100644 index 00000000..a63d1d13 --- /dev/null +++ b/core/tasks/hazmat.py @@ -0,0 +1,19 @@ +import django +from django.core.cache import cache +from django.db import connection + +"""This module is an optimization for dispatcherd workers +This sets up Django pre-fork, which must be implemented as a module to run +on-import for compatibility with multiprocessing forkserver. +This should never be imported by other modules, which is why it is called +hazmat. +""" + + +django.setup() + +# connections may or may not be open, but +# before forking, all connections should be closed + +cache.close() +connection.close() diff --git a/core/views.py b/core/views.py index 20ccf3a4..66132528 100644 --- a/core/views.py +++ b/core/views.py @@ -1,3 +1,5 @@ +import uuid + from ansible_base.lib.utils.views.ansible_base import AnsibleBaseView from rest_framework import status from rest_framework.decorators import api_view @@ -6,6 +8,8 @@ from rest_framework.viewsets import ModelViewSet from rest_framework.viewsets import ReadOnlyModelViewSet +from core.tasks.demo import sumbit_hello_world + from .models import Automation from .models import ControllerLabel from .models import Pattern @@ -94,4 +98,9 @@ def ping(request: Request) -> Response: @api_view(["GET"]) def test(request: Request) -> Response: - return Response(data={"hello": "world"}, status=200) + text = f"hello world from uuid = {uuid.uuid4()}" + id = sumbit_hello_world(text) + return Response( + f"Task submitted (uuid={id}), check dispatcher logs. Should print '{text}'", + status=200, + ) diff --git a/pattern_service/settings/__init__.py b/pattern_service/settings/__init__.py index 65454db8..e004c29d 100644 --- a/pattern_service/settings/__init__.py +++ b/pattern_service/settings/__init__.py @@ -5,6 +5,8 @@ from ansible_base.lib.dynamic_config import load_envvars from ansible_base.lib.dynamic_config import load_standard_settings_files +from .dispatcher import override_dispatcher_settings + try: from dotenv import load_dotenv @@ -24,4 +26,5 @@ ) load_standard_settings_files(DYNACONF) load_envvars(DYNACONF) +override_dispatcher_settings(DYNACONF) export(__name__, DYNACONF) diff --git a/pattern_service/settings/defaults.py b/pattern_service/settings/defaults.py index fdf3c2eb..2c5ecfe9 100644 --- a/pattern_service/settings/defaults.py +++ b/pattern_service/settings/defaults.py @@ -108,3 +108,28 @@ "NAME": BASE_DIR / "db.sqlite3", } } + +DISPATCHER_CONFIG = { + "version": 2, + "service": { + "main_kwargs": {"node_id": "pattern-service-a"}, + "process_manager_kwargs": { + "preload_modules": ["pattern_service.core.tasks.hazmat"] + }, + }, + "brokers": { + "pg_notify": { + "config": { + "conninfo": ( + "dbname=pattern_db user=pattern password=pattern123 " + "host=postgres port=5432 application_name=dispatcher_pattern_service" + ) + }, + "sync_connection_factory": "dispatcherd.brokers.pg_notify.connection_saver", + "channels": ["pattern-service-tasks"], + "default_publish_channel": "pattern-service-tasks", + }, + "socket": {"socket_path": "pattern_service_dispatcher.sock"}, + }, + "publish": {"default_control_broker": "socket", "default_broker": "pg_notify"}, +} diff --git a/pattern_service/settings/development_defaults.py b/pattern_service/settings/development_defaults.py index 7b9760e7..aadd6675 100644 --- a/pattern_service/settings/development_defaults.py +++ b/pattern_service/settings/development_defaults.py @@ -36,5 +36,6 @@ "handlers": ["console"], "level": "INFO", }, + "dispatcherd": {"handlers": ["console"], "level": "INFO"}, }, } diff --git a/pattern_service/settings/dispatcher.py b/pattern_service/settings/dispatcher.py new file mode 100644 index 00000000..2fd7ff31 --- /dev/null +++ b/pattern_service/settings/dispatcher.py @@ -0,0 +1,63 @@ +# Copyright 2025 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.core.exceptions import ImproperlyConfigured +from dynaconf import Dynaconf + + +def override_dispatcher_settings(loaded_settings: Dynaconf) -> None: + databases = loaded_settings.get("DATABASES", {}) + if databases and "default" not in databases: + raise ImproperlyConfigured("DATABASES settings must contain a 'default' key") + + db_host = loaded_settings.get("DB_HOST", "127.0.0.1") + db_port = loaded_settings.get("DB_PORT", 5432) + db_user = loaded_settings.get("DB_USER", "postgres") + db_user_pass = loaded_settings.get("DB_PASSWORD") + db_name = loaded_settings.get("DB_NAME", "pattern_db") + db_app_name = loaded_settings.get("DB_APP_NAME", "dispatcher_pattern_service") + db_sslmode = loaded_settings.get("DB_SSLMODE", default="allow") + db_sslcert = loaded_settings.get("DB_SSLCERT", default="") + db_sslkey = loaded_settings.get("DB_SSLKEY", default="") + db_sslrootcert = loaded_settings.get("DB_SSLROOTCERT", default="") + + databases["default"] = { + "ENGINE": "django.db.backends.postgresql", + "HOST": db_host, + "PORT": db_port, + "USER": db_user, + "PASSWORD": db_user_pass, + "NAME": db_name, + "OPTIONS": { + "sslmode": db_sslmode, + "sslcert": db_sslcert, + "sslkey": db_sslkey, + "sslrootcert": db_sslrootcert, + }, + } + + dispatcher_conninfo = ( + f"dbname={db_name} user={db_user} password={db_user_pass} " + f"host={db_host} port={db_port} application_name={db_app_name}" + ) + dispatcher_node_id = loaded_settings.get("DISPATCHER_NODE_ID", default="") + config = loaded_settings.get("DISPATCHER_CONFIG") + config["brokers"]["pg_notify"]["config"].update({"conninfo": dispatcher_conninfo}) + if dispatcher_node_id: + config["service"]["main_kwargs"]["node_id"] = dispatcher_node_id + + loaded_settings.update( + {"DATABASES": databases, "DISPATCHER_CONFIG": config}, + loader_identifier="settings:override_dispatcher_settings", + ) diff --git a/pattern_service/settings/testing_defaults.py b/pattern_service/settings/testing_defaults.py index 480e4b87..43dbf94a 100644 --- a/pattern_service/settings/testing_defaults.py +++ b/pattern_service/settings/testing_defaults.py @@ -3,3 +3,17 @@ ALLOWED_HOSTS = ["localhost", "pattern-service", "127.0.0.1"] BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = "insecure" + +DB_NAME = "test_pattern_db" +DB_USER = "postgres" +DB_PASSWORD = "insecure" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "localhost", + "PORT": 5432, + "PASSWORD": DB_PASSWORD, + "NAME": DB_NAME, + } +} diff --git a/pyproject.toml b/pyproject.toml index 25c3a110..455689de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ license-files = ["LICENSE.md"] requires-python = ">=3.11,<3.14" dependencies = [ "django-ansible-base==2025.5.8", + "dispatcherd", + "psycopg", ] [project.urls] @@ -30,6 +32,7 @@ Repository = "https://github.com/ansible/pattern-service" dev = [ "pip-tools>=7.4,<8.0", "python-dotenv>=1.1.1,<2.0", + "psycopg-binary==3.2.9", ] test = [ "black>=24.0,<25.0", @@ -72,7 +75,7 @@ disallow_subclassing_any = false disallow_untyped_decorators = false [[tool.mypy.overrides]] -module = ["ansible_base.*", "dotenv.*"] +module = ["ansible_base.*", "dotenv.*", "dispatcherd.*", "dynaconf.*"] ignore_missing_imports = true @@ -82,7 +85,7 @@ ignore_missing_imports = true [tool.tox] requires = ["tox>=4.26"] -env_list = ["check", "format", "lint", "test", "type"] +env_list = ["check", "format", "lint", "type"] [tool.tox.env_run_base] package = "wheel" diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 88abde24..2842d262 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=dev --extra=test --output-file=requirements/requirements-dev.txt pyproject.toml @@ -26,6 +26,8 @@ colorama==0.4.6 # via tox cryptography==45.0.4 # via django-ansible-base +dispatcherd==2025.5.21 + # via pattern_service (pyproject.toml) distlib==0.3.9 # via virtualenv django==4.2.23 @@ -91,6 +93,10 @@ pluggy==1.6.0 # via # pytest # tox +psycopg==3.2.9 + # via pattern_service (pyproject.toml) +psycopg-binary==3.2.9 + # via pattern_service (pyproject.toml) pycodestyle==2.11.1 # via flake8 pycparser==2.22 @@ -111,6 +117,8 @@ pytest-django==4.11.1 # via pattern_service (pyproject.toml) python-dotenv==1.1.1 # via pattern_service (pyproject.toml) +pyyaml==6.0.2 + # via dispatcherd sqlparse==0.5.3 # via # django @@ -124,6 +132,7 @@ typing-extensions==4.13.2 # django-stubs # django-stubs-ext # mypy + # psycopg virtualenv==20.31.2 # via tox wheel==0.45.1 diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index 1ce5fad0..22ed1800 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=test --output-file=requirements/requirements-test.txt pyproject.toml @@ -22,6 +22,8 @@ colorama==0.4.6 # via tox cryptography==45.0.5 # via django-ansible-base +dispatcherd==2025.5.21 + # via pattern_service (pyproject.toml) distlib==0.3.9 # via virtualenv django==4.2.23 @@ -84,6 +86,8 @@ pluggy==1.6.0 # via # pytest # tox +psycopg==3.2.9 + # via pattern_service (pyproject.toml) pycodestyle==2.11.1 # via flake8 pycparser==2.22 @@ -98,6 +102,8 @@ pytest==8.4.1 # via pytest-django pytest-django==4.11.1 # via pattern_service (pyproject.toml) +pyyaml==6.0.2 + # via dispatcherd sqlparse==0.5.3 # via # django @@ -111,5 +117,6 @@ typing-extensions==4.14.1 # django-stubs # django-stubs-ext # mypy + # psycopg virtualenv==20.31.2 # via tox diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 1414f87a..f4a62e14 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/requirements.txt pyproject.toml @@ -10,6 +10,8 @@ cffi==1.17.1 # via cryptography cryptography==45.0.4 # via django-ansible-base +dispatcherd==2025.5.21 + # via pattern_service (pyproject.toml) django==4.2.23 # via # django-ansible-base @@ -25,9 +27,15 @@ dynaconf==3.2.11 # via django-ansible-base inflection==0.5.1 # via django-ansible-base +psycopg==3.2.9 + # via pattern_service (pyproject.toml) pycparser==2.22 # via cffi +pyyaml==6.0.2 + # via dispatcherd sqlparse==0.5.3 # via # django # django-ansible-base +typing-extensions==4.14.1 + # via psycopg diff --git a/tools/podman/Containerfile.dev b/tools/podman/Containerfile.dev new file mode 100644 index 00000000..7663b547 --- /dev/null +++ b/tools/podman/Containerfile.dev @@ -0,0 +1,21 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +WORKDIR /app + +COPY requirements/requirements-dev.txt . + +RUN dnf install python3.11 python3.11-pip -y + +RUN python3.11 -m pip install --no-cache-dir -r requirements-dev.txt + +ADD core /app/core + +ADD pattern_service /app/pattern_service + +COPY manage.py . + +ENV PATTERN_SERVICE_MODE=development + +EXPOSE 5000 + +CMD ["python3.11", "/app/manage.py", "runserver", "0.0.0.0:5000"] diff --git a/tools/podman/README.md b/tools/podman/README.md new file mode 100644 index 00000000..abf4ecc1 --- /dev/null +++ b/tools/podman/README.md @@ -0,0 +1,51 @@ +# Container Compose for Development + +## Getting started + +### Clone the repo + +If you have not already done so, you will need to clone, or create a local copy, of the [pattern-service repo](https://github.com/ansible/pattern-service). +Once you have a local copy, run the commands in the following sections from the root of the project tree. + +### Prerequisites + +- [Podman](https://podman.io/docs/installation) on the host where the application will be deployed. +- [Podman Compose](https://podman-desktop.io/docs/compose/setting-up-compose) or [python installation](https://pypi.org/project/podman-compose/). + + +## Starting the Development Environment + +### Build the Image + +Run the following to build the image: + +```bash +$ make compose-build +``` + +> The image will need to be rebuilt if there are any changes to `tools/podman/compose.yaml`. + +Once the build completes, you will have a `ansible/awx_devel` image in your local image cache. Use the `podman images` command to view it, as follows: + +```bash +(host)$ podman images + +REPOSITORY TAG IMAGE ID CREATED SIZE +localhost/pattern-service-api latest fcf098365c6a 2 minutes ago 739MB +localhost/pattern-service-worker latest 20e63a95799b 2 minutes ago 739MB +quay.io/sclorg/postgresql-15-c9s latest 8e0c195e634c 2 minutes ago 372MB +``` + +### Run the pattern-service application + +##### Start the containers + +Run the pattern-service-api, pattern-service-worker, and postgres containers. This utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, you'll be able to watch log messages and events in real time. + +```bash +$ make compose-up +``` + +> For running podman-compose in detached mode, start the containers using the following command: `$ make compose-up COMPOSE_UP_OPTS=-d` + +You can test the application from the url `http://localhost:8000/api/pattern-service/v1/test/` diff --git a/tools/podman/compose-test.yaml b/tools/podman/compose-test.yaml new file mode 100644 index 00000000..7761312f --- /dev/null +++ b/tools/podman/compose-test.yaml @@ -0,0 +1,7 @@ +services: + postgres: + image: quay.io/sclorg/postgresql-15-c9s:latest + environment: + POSTGRESQL_ADMIN_PASSWORD: insecure + ports: + - '5432:5432' diff --git a/tools/podman/compose.yaml b/tools/podman/compose.yaml new file mode 100644 index 00000000..0b2dec10 --- /dev/null +++ b/tools/podman/compose.yaml @@ -0,0 +1,72 @@ +x-environment: &common-env + PATTERN_SERVICE_DB_HOST: postgres + PATTERN_SERVICE_DB_PORT: 5432 + PATTERN_SERVICE_DB_NAME: pattern_db + PATTERN_SERVICE_DB_USER: pattern + PATTERN_SERVICE_DB_PASSWORD: pattern123 + PATTERN_SERVICE_MODE: development +services: + postgres: + image: quay.io/sclorg/postgresql-15-c9s:latest + environment: + POSTGRESQL_USER: pattern + POSTGRESQL_PASSWORD: pattern123 + POSTGRESQL_DATABASE: pattern_db + ports: + - '5432:5432' + volumes: + - 'postgres_data:/var/lib/pgsql/data' + healthcheck: + test: ["CMD", "pg_isready", "-U", "pattern", "-d", "pattern_db"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + + pattern-service-worker: + image: localhost/pattern-service-worker + environment: + << : *common-env + PATTERN_SERVICE_DISPATCHER_NODE_ID: pattern-service-worker + build: + context: ../../ + dockerfile: tools/podman/Containerfile.dev + command: + - /bin/bash + - -c + - python3.11 /app/manage.py worker + depends_on: + postgres: + condition: service_healthy + restart: always + + pattern-service-api: + image: localhost/pattern-service-api + environment: + << : *common-env + PATTERN_SERVICE_DISPATCHER_NODE_ID: pattern-service-api + build: + context: ../../ + dockerfile: tools/podman/Containerfile.dev + command: + - /bin/bash + - -c + - >- + python3.11 /app/manage.py makemigrations + && python3.11 /app/manage.py migrate + && python3.11 /app/manage.py runserver 0.0.0.0:5000 + ports: + - "8000:5000" + depends_on: + postgres: + condition: service_healthy + pattern-service-worker: + condition: service_started + healthcheck: + test: [ 'CMD', 'curl', '-q', 'http://localhost:8000/ping/' ] + interval: 30s + timeout: 5s + retries: 10 + +volumes: + postgres_data: {}