From aa00ef01aa42feeee446160a8bb6eb162575a4a1 Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Tue, 15 Jul 2025 11:45:48 -0400 Subject: [PATCH 1/8] Set up docker for local development --- .dockerignore | 1 + Dockerfile.dev | 21 ------- Makefile | 28 ++++++++- core/apps.py | 15 +++++ core/management/__init__.py | 0 core/management/commands/__init__.py | 0 core/management/commands/worker.py | 14 +++++ core/tasks/__init__.py | 0 core/tasks/hazmat.py | 19 ++++++ pattern_service/settings/__init__.py | 3 + pattern_service/settings/defaults.py | 30 +++++++++ pattern_service/settings/dispatcher.py | 75 +++++++++++++++++++++++ pyproject.toml | 5 +- requirements/requirements-dev.txt | 11 +++- requirements/requirements-test.txt | 9 +++ requirements/requirements.txt | 10 +++ tools/docker/Dockerfile.dev | 23 +++++++ tools/docker/README.md | 51 ++++++++++++++++ tools/docker/docker-compose.yaml | 84 ++++++++++++++++++++++++++ 19 files changed, 375 insertions(+), 24 deletions(-) create mode 100644 .dockerignore delete mode 100644 Dockerfile.dev create mode 100644 core/management/__init__.py create mode 100644 core/management/commands/__init__.py create mode 100644 core/management/commands/worker.py create mode 100644 core/tasks/__init__.py create mode 100644 core/tasks/hazmat.py create mode 100644 pattern_service/settings/dispatcher.py create mode 100644 tools/docker/Dockerfile.dev create mode 100644 tools/docker/README.md create mode 100644 tools/docker/docker-compose.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a807db44 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +**/db.sqlite3 diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 3453f514..00000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,21 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi:latest - -WORKDIR /app - -COPY requirements/requirements.txt . - -RUN dnf install python3-pip -y - -RUN python3 -m pip install --no-cache-dir -r requirements.txt - -ADD core /app/core - -ADD pattern_service /app/pattern_service - -COPY manage.py . - -RUN python3 manage.py migrate - -EXPOSE 5000 - -CMD ["python3", "/app/manage.py", "runserver", "0.0.0.0:5000"] diff --git a/Makefile b/Makefile index 5f21cd8f..f7bc7359 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,8 @@ help: ## Show this help message CONTAINER_RUNTIME ?= podman IMAGE_NAME ?= pattern-service IMAGE_TAG ?= latest +QUAY_NAMESPACE ?= ansible +BUILD_ARGS ?= "--arch amd64" ensure-namespace: @test -n "$$QUAY_NAMESPACE" || (echo "Error: QUAY_NAMESPACE is required to push quay.io" && exit 1) @@ -21,7 +23,7 @@ ensure-namespace: .PHONY: build build: ## Build the container image @echo "Building container image..." - $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f Dockerfile.dev --arch amd64 . + $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f tools/docker/Dockerfile.dev $(BUILD_ARGS) . .PHONY: clean clean: ## Remove container image @@ -34,6 +36,30 @@ push: ensure-namespace build ## Tag and push container image to Quay.io $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):$(IMAGE_TAG) quay.io/$(QUAY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG) $(CONTAINER_RUNTIME) push quay.io/$(QUAY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG) +#-------------------------------------- +# Compose +# ------------------------------------- +.PHONY: compose-build +compose-build: + $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml $(COMPOSE_OPTS) build + +.PHONY: compose-up +compose-up: + $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans + +.PHONY: compose-down +compose-down: + $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml $(COMPOSE_OPTS) down --remove-orphans + +.PHONY: compose-clean +compose-clean: + $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml rm -sf + docker rmi --force localhost/ansible-pattern-service-api localhost/ansible-pattern-service-worker + docker volume rm -f postgres_data + +.PHONY: compose-restart +compose-restart: compose-down compose-clean compose-up + # ------------------------------------- # Dependencies # ------------------------------------- diff --git a/core/apps.py b/core/apps.py index c0ce093b..2a565500 100644 --- a/core/apps.py +++ b/core/apps.py @@ -1,6 +1,21 @@ +import logging +import os + +from dispatcherd.config import setup as dispatcher_setup from django.apps import AppConfig +from django.conf import settings + +logger = logging.getLogger(__name__) class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "core" + + def ready(self) -> None: + dispatcher_feature = os.environ.get( + "PATTERN_SERVICE_FEATURE_DISPATCHERD", "false" + ) + if dispatcher_feature.lower() in ("true", "1", "yes"): + # Setup dispatcher configuration + dispatcher_setup(config=settings.DISPATCHER_CONFIG) diff --git a/core/management/__init__.py b/core/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/management/commands/__init__.py b/core/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/management/commands/worker.py b/core/management/commands/worker.py new file mode 100644 index 00000000..72763de5 --- /dev/null +++ b/core/management/commands/worker.py @@ -0,0 +1,14 @@ +import logging + +from dispatcherd import run_service +from django.core.management.base import BaseCommand + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """Wrapper for worker command.""" + + def handle(self, *args: tuple, **options: dict) -> None: + logger.info("Starting Pattern service dispatcherd worker.") + run_service() diff --git a/core/tasks/__init__.py b/core/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/tasks/hazmat.py b/core/tasks/hazmat.py new file mode 100644 index 00000000..a63d1d13 --- /dev/null +++ b/core/tasks/hazmat.py @@ -0,0 +1,19 @@ +import django +from django.core.cache import cache +from django.db import connection + +"""This module is an optimization for dispatcherd workers +This sets up Django pre-fork, which must be implemented as a module to run +on-import for compatibility with multiprocessing forkserver. +This should never be imported by other modules, which is why it is called +hazmat. +""" + + +django.setup() + +# connections may or may not be open, but +# before forking, all connections should be closed + +cache.close() +connection.close() diff --git a/pattern_service/settings/__init__.py b/pattern_service/settings/__init__.py index 65454db8..e004c29d 100644 --- a/pattern_service/settings/__init__.py +++ b/pattern_service/settings/__init__.py @@ -5,6 +5,8 @@ from ansible_base.lib.dynamic_config import load_envvars from ansible_base.lib.dynamic_config import load_standard_settings_files +from .dispatcher import override_dispatcher_settings + try: from dotenv import load_dotenv @@ -24,4 +26,5 @@ ) load_standard_settings_files(DYNACONF) load_envvars(DYNACONF) +override_dispatcher_settings(DYNACONF) export(__name__, DYNACONF) diff --git a/pattern_service/settings/defaults.py b/pattern_service/settings/defaults.py index fdf3c2eb..9eeb8991 100644 --- a/pattern_service/settings/defaults.py +++ b/pattern_service/settings/defaults.py @@ -108,3 +108,33 @@ "NAME": BASE_DIR / "db.sqlite3", } } + +DISPATCHER_CONFIG = { + "version": 2, + "service": { + "pool_kwargs": {"min_workers": 2, "max_workers": 12, "scaledown_wait": 15}, + "main_kwargs": {"node_id": "pattern-service-a"}, + "process_manager_cls": "ForkServerManager", + "process_manager_kwargs": { + "preload_modules": ["pattern_service.core.tasks.hazmat"] + }, + }, + "worker": {"worker_kwargs": {"idle_timeout": 3}}, + "brokers": { + "pg_notify": { + "config": { + "conninfo": ( + "dbname=pattern_db user=pattern password=pattern123 " + "host=postgres port=5432 application_name=dispatcher_pattern_service" + ) + }, + "sync_connection_factory": "dispatcherd.brokers.pg_notify.connection_saver", + "channels": ["pattern-service-tasks"], + "default_publish_channel": "pattern-service-tasks", + "max_connection_idle_seconds": 5, + "max_self_check_message_age_seconds": 2, + }, + "socket": {"socket_path": "pattern_service_dispatcher.sock"}, + }, + "publish": {"default_control_broker": "socket", "default_broker": "pg_notify"}, +} diff --git a/pattern_service/settings/dispatcher.py b/pattern_service/settings/dispatcher.py new file mode 100644 index 00000000..07f9b489 --- /dev/null +++ b/pattern_service/settings/dispatcher.py @@ -0,0 +1,75 @@ +# Copyright 2025 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.core.exceptions import ImproperlyConfigured +from dynaconf import Dynaconf + + +def convert_to_bool(value: str) -> bool: + if isinstance(value, bool): + return value + return str(value).lower() in ("yes", "true", "1") + + +def override_dispatcher_settings(loaded_settings: Dynaconf) -> None: + feature_dispatcherd = convert_to_bool(loaded_settings.get("FEATURE_DISPATCHERD")) + if feature_dispatcherd: + databases = loaded_settings.get("DATABASES", {}) + if databases and "default" not in databases: + raise ImproperlyConfigured( + "DATABASES settings must contain a 'default' key" + ) + + db_host = loaded_settings.get("DB_HOST", "127.0.0.1") + db_port = loaded_settings.get("DB_PORT", 5432) + db_user = loaded_settings.get("DB_USER", "postgres") + db_user_pass = loaded_settings.get("DB_PASSWORD") + db_name = loaded_settings.get("DB_NAME", "pattern_db") + db_app_name = loaded_settings.get("DB_APP_NAME", "dispatcher_pattern_service") + db_sslmode = loaded_settings.get("DB_SSLMODE", default="allow") + db_sslcert = loaded_settings.get("DB_SSLCERT", default="") + db_sslkey = loaded_settings.get("DB_SSLKEY", default="") + db_sslrootcert = loaded_settings.get("DB_SSLROOTCERT", default="") + + databases["default"] = { + "ENGINE": "django.db.backends.postgresql", + "HOST": db_host, + "PORT": db_port, + "USER": db_user, + "PASSWORD": db_user_pass, + "NAME": db_name, + "OPTIONS": { + "sslmode": db_sslmode, + "sslcert": db_sslcert, + "sslkey": db_sslkey, + "sslrootcert": db_sslrootcert, + }, + } + + dispatcher_conninfo = ( + f"dbname={db_name} user={db_user} password={db_user_pass} " + f"host={db_host} port={db_port} application_name={db_app_name}" + ) + dispatcher_node_id = loaded_settings.get("DISPATCHER_NODE_ID", default="") + config = loaded_settings.get("DISPATCHER_CONFIG") + config["brokers"]["pg_notify"]["config"].update( + {"conninfo": dispatcher_conninfo} + ) + if dispatcher_node_id: + config["service"]["main_kwargs"]["node_id"] = dispatcher_node_id + + loaded_settings.update( + {"DATABASES": databases, "DISPATCHER_CONFIG": config}, + loader_identifier="settings:override_dispatcher_settings", + ) diff --git a/pyproject.toml b/pyproject.toml index 25c3a110..22d74f78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ license-files = ["LICENSE.md"] requires-python = ">=3.11,<3.14" dependencies = [ "django-ansible-base==2025.5.8", + "dispatcherd", + "psycopg", + "psycopg-binary", ] [project.urls] @@ -72,7 +75,7 @@ disallow_subclassing_any = false disallow_untyped_decorators = false [[tool.mypy.overrides]] -module = ["ansible_base.*", "dotenv.*"] +module = ["ansible_base.*", "dotenv.*", "dispatcherd.*", "dynaconf.*"] ignore_missing_imports = true diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 88abde24..2842d262 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=dev --extra=test --output-file=requirements/requirements-dev.txt pyproject.toml @@ -26,6 +26,8 @@ colorama==0.4.6 # via tox cryptography==45.0.4 # via django-ansible-base +dispatcherd==2025.5.21 + # via pattern_service (pyproject.toml) distlib==0.3.9 # via virtualenv django==4.2.23 @@ -91,6 +93,10 @@ pluggy==1.6.0 # via # pytest # tox +psycopg==3.2.9 + # via pattern_service (pyproject.toml) +psycopg-binary==3.2.9 + # via pattern_service (pyproject.toml) pycodestyle==2.11.1 # via flake8 pycparser==2.22 @@ -111,6 +117,8 @@ pytest-django==4.11.1 # via pattern_service (pyproject.toml) python-dotenv==1.1.1 # via pattern_service (pyproject.toml) +pyyaml==6.0.2 + # via dispatcherd sqlparse==0.5.3 # via # django @@ -124,6 +132,7 @@ typing-extensions==4.13.2 # django-stubs # django-stubs-ext # mypy + # psycopg virtualenv==20.31.2 # via tox wheel==0.45.1 diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index 1ce5fad0..3724ab99 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -22,6 +22,8 @@ colorama==0.4.6 # via tox cryptography==45.0.5 # via django-ansible-base +dispatcherd==2025.5.21 + # via pattern_service (pyproject.toml) distlib==0.3.9 # via virtualenv django==4.2.23 @@ -84,6 +86,10 @@ pluggy==1.6.0 # via # pytest # tox +psycopg==3.2.9 + # via pattern_service (pyproject.toml) +psycopg-binary==3.2.9 + # via pattern_service (pyproject.toml) pycodestyle==2.11.1 # via flake8 pycparser==2.22 @@ -98,6 +104,8 @@ pytest==8.4.1 # via pytest-django pytest-django==4.11.1 # via pattern_service (pyproject.toml) +pyyaml==6.0.2 + # via dispatcherd sqlparse==0.5.3 # via # django @@ -111,5 +119,6 @@ typing-extensions==4.14.1 # django-stubs # django-stubs-ext # mypy + # psycopg virtualenv==20.31.2 # via tox diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 1414f87a..da9b7939 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -10,6 +10,8 @@ cffi==1.17.1 # via cryptography cryptography==45.0.4 # via django-ansible-base +dispatcherd==2025.5.21 + # via pattern_service (pyproject.toml) django==4.2.23 # via # django-ansible-base @@ -25,9 +27,17 @@ dynaconf==3.2.11 # via django-ansible-base inflection==0.5.1 # via django-ansible-base +psycopg==3.2.9 + # via pattern_service (pyproject.toml) +psycopg-binary==3.2.9 + # via pattern_service (pyproject.toml) pycparser==2.22 # via cffi +pyyaml==6.0.2 + # via dispatcherd sqlparse==0.5.3 # via # django # django-ansible-base +typing-extensions==4.14.1 + # via psycopg diff --git a/tools/docker/Dockerfile.dev b/tools/docker/Dockerfile.dev new file mode 100644 index 00000000..d4706cb5 --- /dev/null +++ b/tools/docker/Dockerfile.dev @@ -0,0 +1,23 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +WORKDIR /app + +COPY requirements/requirements.txt . + +RUN dnf install python3.11 python3.11-pip -y + +RUN python3.11 -m pip install --no-cache-dir -r requirements.txt + +ADD core /app/core + +ADD pattern_service /app/pattern_service + +COPY manage.py . + +ENV PATTERN_SERVICE_MODE=development + +RUN python3.11 manage.py migrate + +EXPOSE 5000 + +CMD ["python3.11", "/app/manage.py", "runserver", "0.0.0.0:5000"] diff --git a/tools/docker/README.md b/tools/docker/README.md new file mode 100644 index 00000000..e5f3605e --- /dev/null +++ b/tools/docker/README.md @@ -0,0 +1,51 @@ +# Container Compose for Development + +## Getting started + +### Clone the repo + +If you have not already done so, you will need to clone, or create a local copy, of the [pattern-service repo](https://github.com/ansible/pattern-service). +Once you have a local copy, run the commands in the following sections from the root of the project tree. + +### Prerequisites + +- [Docker](https://docs.docker.com/engine/installation/) on the host where the application will be deployed. After installing Docker, the Docker service must be started (depending on your OS, you may have to add the local user that uses Docker to the `docker` group, refer to the documentation for details) +- [Docker Compose](https://docs.docker.com/compose/install/). + + +## Starting the Development Environment + +### Build the Image + +Run the following to build the image: + +```bash +$ make compose-build +``` + +> The image will need to be rebuilt if there are any changes to `tools/docker/docker-compose.yaml`. + +Once the build completes, you will have a `ansible/awx_devel` image in your local image cache. Use the `docker images` command to view it, as follows: + +```bash +(host)$ docker images + +REPOSITORY TAG IMAGE ID CREATED SIZE +localhost/ansible-pattern-service-api latest fcf098365c6a 2 minutes ago 739MB +localhost/ansible-pattern-service-worker latest 20e63a95799b 2 minutes ago 739MB +quay.io/sclorg/postgresql-15-c9s latest 8e0c195e634c 2 minutes ago 372MB +``` + +### Run the pattern-service application + +##### Start the containers + +Run the pattern-service-api, pattern-service-worker, and postgres containers. This utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, your session will be attached to the awx container, and you'll be able to watch log messages and events in real time. You will see messages from Django and the build process. + +```bash +$ make compose-up +``` + +> For running docker-compose detached mode, start the containers using the following command: `$ make compose-up COMPOSE_UP_OPTS=-d` + +You can test the application from the url `http://localhost:8000/api/pattern-service/v1/test/` diff --git a/tools/docker/docker-compose.yaml b/tools/docker/docker-compose.yaml new file mode 100644 index 00000000..09298383 --- /dev/null +++ b/tools/docker/docker-compose.yaml @@ -0,0 +1,84 @@ +x-environment: &common-env + PATTERN_SERVICE_DB_HOST: postgres + PATTERN_SERVICE_DB_PORT: 5432 + PATTERN_SERVICE_DB_NAME: pattern_db + PATTERN_SERVICE_DB_USER: pattern + PATTERN_SERVICE_DB_PASSWORD: pattern123 + PATTERN_SERVICE_FEATURE_DISPATCHERD: "true" + PATTERN_SERVICE_MODE: development +services: + postgres: + image: quay.io/sclorg/postgresql-15-c9s:latest + environment: + POSTGRESQL_USER: pattern + POSTGRESQL_PASSWORD: pattern123 + POSTGRESQL_DATABASE: pattern_db + ports: + - '5432:5432' + volumes: + - 'postgres_data:/var/lib/pgsql/data' + healthcheck: + test: ["CMD", "pg_isready", "-U", "pattern", "-d", "pattern_db"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + networks: + - dev + + pattern-service-worker: + image: localhost/ansible-pattern-service-worker + environment: + << : *common-env + PATTERN_SERVICE_DISPATCHER_NODE_ID: pattern-service-worker + build: + context: ../../ + dockerfile: tools/docker/Dockerfile.dev + command: + - /bin/bash + - -c + - python3.11 /app/manage.py worker + depends_on: + postgres: + condition: service_healthy + restart: always + networks: + - dev + + pattern-service-api: + image: localhost/ansible-pattern-service-api + environment: + << : *common-env + PATTERN_SERVICE_DISPATCHER_NODE_ID: pattern-service-api + build: + context: ../../ + dockerfile: tools/docker/Dockerfile.dev + command: + - /bin/bash + - -c + - >- + python3.11 /app/manage.py makemigrations core + && python3.11 /app/manage.py migrate core + && python3.11 /app/manage.py runserver 0.0.0.0:5000 + ports: + - "8000:5000" + depends_on: + postgres: + condition: service_healthy + pattern-service-worker: + condition: service_started + healthcheck: + test: [ 'CMD', 'curl', '-q', 'http://localhost:8000/ping/' ] + interval: 30s + timeout: 5s + retries: 10 + networks: + - dev + - default + +volumes: + postgres_data: {} + +networks: + dev: + name: dev From b995143b15fc96d231a6b29d2b6dd6f2077ca9eb Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 30 Jul 2025 11:26:32 +0200 Subject: [PATCH 2/8] Code review updates --- .dockerignore | 84 ++++++++++++++++++- Makefile | 24 ++---- pattern_service/settings/defaults.py | 5 -- pyproject.toml | 2 +- requirements/requirements-test.txt | 4 +- requirements/requirements.txt | 4 +- .../Containerfile.dev} | 4 +- tools/{docker => podman}/README.md | 14 ++-- .../compose.yaml} | 18 +--- 9 files changed, 108 insertions(+), 51 deletions(-) rename tools/{docker/Dockerfile.dev => podman/Containerfile.dev} (75%) rename tools/{docker => podman}/README.md (66%) rename tools/{docker/docker-compose.yaml => podman/compose.yaml} (84%) diff --git a/.dockerignore b/.dockerignore index a807db44..27293dc0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,83 @@ -**/db.sqlite3 +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Virtual environment +.venv/ +venv/ +env/ + +# IDE and Editor files +.vscode/ +.idea/ +*.swp +*.bak +*.swo + +# Gitignore and other version control files +.git/ +.gitignore +.gitmodules + +# Environment variables +.env +.env.* +*.env + +# Logs and temporary files +*.log +tmp/ +temp/ + +# Database files +*.sqlite3 +*.db + +# Testing +.cache +.coverage +.tox +coverage.xml +htmlcov +pep8.txt +scratch +testem.log +.pytest_cache/ + +# Mac OS X +*.DS_Store + +# VSCode +.vscode/ \ No newline at end of file diff --git a/Makefile b/Makefile index f7bc7359..f0ad0a7b 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ CONTAINER_RUNTIME ?= podman IMAGE_NAME ?= pattern-service IMAGE_TAG ?= latest QUAY_NAMESPACE ?= ansible -BUILD_ARGS ?= "--arch amd64" +BUILD_ARGS ?= --arch amd64 ensure-namespace: @test -n "$$QUAY_NAMESPACE" || (echo "Error: QUAY_NAMESPACE is required to push quay.io" && exit 1) @@ -23,7 +23,7 @@ ensure-namespace: .PHONY: build build: ## Build the container image @echo "Building container image..." - $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f tools/docker/Dockerfile.dev $(BUILD_ARGS) . + $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f tools/podman/Containerfile.dev $(BUILD_ARGS) . .PHONY: clean clean: ## Remove container image @@ -40,25 +40,19 @@ push: ensure-namespace build ## Tag and push container image to Quay.io # Compose # ------------------------------------- .PHONY: compose-build -compose-build: - $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml $(COMPOSE_OPTS) build +compose-build: ## Build the containers images for the services + $(CONTAINER_RUNTIME) compose -f tools/podman/compose.yaml $(COMPOSE_OPTS) build -.PHONY: compose-up +.PHONY: compose-up ## Build and start the containers for the services compose-up: - $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans + $(CONTAINER_RUNTIME) compose -f tools/podman/compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans .PHONY: compose-down -compose-down: - $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml $(COMPOSE_OPTS) down --remove-orphans - -.PHONY: compose-clean -compose-clean: - $(CONTAINER_RUNTIME) compose -f tools/docker/docker-compose.yaml rm -sf - docker rmi --force localhost/ansible-pattern-service-api localhost/ansible-pattern-service-worker - docker volume rm -f postgres_data +compose-down: ## Stop containers and remove containers, network, images and volumes created by compose-up + $(CONTAINER_RUNTIME) compose -f tools/podman/compose.yaml $(COMPOSE_OPTS) down --remove-orphans .PHONY: compose-restart -compose-restart: compose-down compose-clean compose-up +compose-restart: compose-down compose-up ## Stop and remove existing infrastructure and start a new one # ------------------------------------- # Dependencies diff --git a/pattern_service/settings/defaults.py b/pattern_service/settings/defaults.py index 9eeb8991..2c5ecfe9 100644 --- a/pattern_service/settings/defaults.py +++ b/pattern_service/settings/defaults.py @@ -112,14 +112,11 @@ DISPATCHER_CONFIG = { "version": 2, "service": { - "pool_kwargs": {"min_workers": 2, "max_workers": 12, "scaledown_wait": 15}, "main_kwargs": {"node_id": "pattern-service-a"}, - "process_manager_cls": "ForkServerManager", "process_manager_kwargs": { "preload_modules": ["pattern_service.core.tasks.hazmat"] }, }, - "worker": {"worker_kwargs": {"idle_timeout": 3}}, "brokers": { "pg_notify": { "config": { @@ -131,8 +128,6 @@ "sync_connection_factory": "dispatcherd.brokers.pg_notify.connection_saver", "channels": ["pattern-service-tasks"], "default_publish_channel": "pattern-service-tasks", - "max_connection_idle_seconds": 5, - "max_self_check_message_age_seconds": 2, }, "socket": {"socket_path": "pattern_service_dispatcher.sock"}, }, diff --git a/pyproject.toml b/pyproject.toml index 22d74f78..ff207b80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ dependencies = [ "django-ansible-base==2025.5.8", "dispatcherd", "psycopg", - "psycopg-binary", ] [project.urls] @@ -33,6 +32,7 @@ Repository = "https://github.com/ansible/pattern-service" dev = [ "pip-tools>=7.4,<8.0", "python-dotenv>=1.1.1,<2.0", + "psycopg-binary==3.2.9", ] test = [ "black>=24.0,<25.0", diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index 3724ab99..22ed1800 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=test --output-file=requirements/requirements-test.txt pyproject.toml @@ -88,8 +88,6 @@ pluggy==1.6.0 # tox psycopg==3.2.9 # via pattern_service (pyproject.toml) -psycopg-binary==3.2.9 - # via pattern_service (pyproject.toml) pycodestyle==2.11.1 # via flake8 pycparser==2.22 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index da9b7939..f4a62e14 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.13 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/requirements.txt pyproject.toml @@ -29,8 +29,6 @@ inflection==0.5.1 # via django-ansible-base psycopg==3.2.9 # via pattern_service (pyproject.toml) -psycopg-binary==3.2.9 - # via pattern_service (pyproject.toml) pycparser==2.22 # via cffi pyyaml==6.0.2 diff --git a/tools/docker/Dockerfile.dev b/tools/podman/Containerfile.dev similarity index 75% rename from tools/docker/Dockerfile.dev rename to tools/podman/Containerfile.dev index d4706cb5..d01cc194 100644 --- a/tools/docker/Dockerfile.dev +++ b/tools/podman/Containerfile.dev @@ -2,11 +2,11 @@ FROM registry.access.redhat.com/ubi9/ubi:latest WORKDIR /app -COPY requirements/requirements.txt . +COPY requirements/requirements-dev.txt . RUN dnf install python3.11 python3.11-pip -y -RUN python3.11 -m pip install --no-cache-dir -r requirements.txt +RUN python3.11 -m pip install --no-cache-dir -r requirements-dev.txt ADD core /app/core diff --git a/tools/docker/README.md b/tools/podman/README.md similarity index 66% rename from tools/docker/README.md rename to tools/podman/README.md index e5f3605e..fd4a5f31 100644 --- a/tools/docker/README.md +++ b/tools/podman/README.md @@ -23,16 +23,16 @@ Run the following to build the image: $ make compose-build ``` -> The image will need to be rebuilt if there are any changes to `tools/docker/docker-compose.yaml`. +The image will need to be rebuilt if there are any changes to `tools/docker/docker-compose.yaml`. -Once the build completes, you will have a `ansible/awx_devel` image in your local image cache. Use the `docker images` command to view it, as follows: +Once the build completes, you will have a few new images in your local image cache. Use the `podman images` command to view them, as follows: ```bash -(host)$ docker images +(host)$ podman images REPOSITORY TAG IMAGE ID CREATED SIZE -localhost/ansible-pattern-service-api latest fcf098365c6a 2 minutes ago 739MB -localhost/ansible-pattern-service-worker latest 20e63a95799b 2 minutes ago 739MB +localhost/pattern-service-api latest fcf098365c6a 2 minutes ago 739MB +localhost/pattern-service-worker latest 20e63a95799b 2 minutes ago 739MB quay.io/sclorg/postgresql-15-c9s latest 8e0c195e634c 2 minutes ago 372MB ``` @@ -40,12 +40,12 @@ quay.io/sclorg/postgresql-15-c9s latest 8e0c195e634c 2 minu ##### Start the containers -Run the pattern-service-api, pattern-service-worker, and postgres containers. This utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, your session will be attached to the awx container, and you'll be able to watch log messages and events in real time. You will see messages from Django and the build process. +Run the pattern-service-api, pattern-service-worker, and postgres containers. This utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, you'll be able to watch log messages and events in real time. ```bash $ make compose-up ``` -> For running docker-compose detached mode, start the containers using the following command: `$ make compose-up COMPOSE_UP_OPTS=-d` +For running docker-compose detached mode, start the containers using the following command: `$ make compose-up COMPOSE_UP_OPTS=-d` You can test the application from the url `http://localhost:8000/api/pattern-service/v1/test/` diff --git a/tools/docker/docker-compose.yaml b/tools/podman/compose.yaml similarity index 84% rename from tools/docker/docker-compose.yaml rename to tools/podman/compose.yaml index 09298383..90f2d596 100644 --- a/tools/docker/docker-compose.yaml +++ b/tools/podman/compose.yaml @@ -23,17 +23,15 @@ services: timeout: 5s retries: 3 start_period: 5s - networks: - - dev pattern-service-worker: - image: localhost/ansible-pattern-service-worker + image: localhost/pattern-service-worker environment: << : *common-env PATTERN_SERVICE_DISPATCHER_NODE_ID: pattern-service-worker build: context: ../../ - dockerfile: tools/docker/Dockerfile.dev + dockerfile: tools/podman/Containerfile.dev command: - /bin/bash - -c @@ -42,17 +40,15 @@ services: postgres: condition: service_healthy restart: always - networks: - - dev pattern-service-api: - image: localhost/ansible-pattern-service-api + image: localhost/pattern-service-api environment: << : *common-env PATTERN_SERVICE_DISPATCHER_NODE_ID: pattern-service-api build: context: ../../ - dockerfile: tools/docker/Dockerfile.dev + dockerfile: tools/podman/Containerfile.dev command: - /bin/bash - -c @@ -72,13 +68,7 @@ services: interval: 30s timeout: 5s retries: 10 - networks: - - dev - - default volumes: postgres_data: {} -networks: - dev: - name: dev From 4076fa4c128da763494a39021f2a0e4a8eba74b8 Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 30 Jul 2025 15:01:53 +0200 Subject: [PATCH 3/8] Fix check --- .dockerignore | 2 +- tools/podman/compose.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 27293dc0..cd02e8a9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -80,4 +80,4 @@ testem.log *.DS_Store # VSCode -.vscode/ \ No newline at end of file +.vscode/ diff --git a/tools/podman/compose.yaml b/tools/podman/compose.yaml index 90f2d596..0872476e 100644 --- a/tools/podman/compose.yaml +++ b/tools/podman/compose.yaml @@ -71,4 +71,3 @@ services: volumes: postgres_data: {} - From 1742ef187f1bbe273771bd6f742356b23e72c20e Mon Sep 17 00:00:00 2001 From: aubin bikouo Date: Fri, 1 Aug 2025 09:30:29 +0200 Subject: [PATCH 4/8] More code review updates --- Makefile | 8 +- core/apps.py | 9 +- core/tasks/demo.py | 18 ++++ core/views.py | 11 +- .../settings/development_defaults.py | 1 + pattern_service/settings/dispatcher.py | 100 ++++++++---------- tools/podman/Containerfile.dev | 2 - tools/podman/README.md | 10 +- tools/podman/compose.yaml | 1 - 9 files changed, 84 insertions(+), 76 deletions(-) create mode 100644 core/tasks/demo.py diff --git a/Makefile b/Makefile index f0ad0a7b..fbbed511 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ help: ## Show this help message # Container image # ------------------------------------- -CONTAINER_RUNTIME ?= podman +COMPOSE_COMMAND ?= podman compose IMAGE_NAME ?= pattern-service IMAGE_TAG ?= latest QUAY_NAMESPACE ?= ansible @@ -41,15 +41,15 @@ push: ensure-namespace build ## Tag and push container image to Quay.io # ------------------------------------- .PHONY: compose-build compose-build: ## Build the containers images for the services - $(CONTAINER_RUNTIME) compose -f tools/podman/compose.yaml $(COMPOSE_OPTS) build + $(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) build .PHONY: compose-up ## Build and start the containers for the services compose-up: - $(CONTAINER_RUNTIME) compose -f tools/podman/compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans + $(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans .PHONY: compose-down compose-down: ## Stop containers and remove containers, network, images and volumes created by compose-up - $(CONTAINER_RUNTIME) compose -f tools/podman/compose.yaml $(COMPOSE_OPTS) down --remove-orphans + $(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) down --remove-orphans .PHONY: compose-restart compose-restart: compose-down compose-up ## Stop and remove existing infrastructure and start a new one diff --git a/core/apps.py b/core/apps.py index 2a565500..169764e9 100644 --- a/core/apps.py +++ b/core/apps.py @@ -1,5 +1,4 @@ import logging -import os from dispatcherd.config import setup as dispatcher_setup from django.apps import AppConfig @@ -13,9 +12,5 @@ class CoreConfig(AppConfig): name = "core" def ready(self) -> None: - dispatcher_feature = os.environ.get( - "PATTERN_SERVICE_FEATURE_DISPATCHERD", "false" - ) - if dispatcher_feature.lower() in ("true", "1", "yes"): - # Setup dispatcher configuration - dispatcher_setup(config=settings.DISPATCHER_CONFIG) + # Configure dispatcher + dispatcher_setup(config=settings.DISPATCHER_CONFIG) diff --git a/core/tasks/demo.py b/core/tasks/demo.py new file mode 100644 index 00000000..338ba0f8 --- /dev/null +++ b/core/tasks/demo.py @@ -0,0 +1,18 @@ +from dispatcherd.publish import submit_task +from dispatcherd.publish import task + +DISPATCHERD_DEFAULT_CHANNEL = "pattern-service-tasks" + + +@task(queue=DISPATCHERD_DEFAULT_CHANNEL, decorate=False) +def print_text(text: str) -> None: + print(text) + + +def sumbit_hello_world(text: str): # type: ignore + job_data, queue = submit_task( + print_text, + queue=DISPATCHERD_DEFAULT_CHANNEL, + args=(text,), + ) + return job_data["uuid"] diff --git a/core/views.py b/core/views.py index 20ccf3a4..66132528 100644 --- a/core/views.py +++ b/core/views.py @@ -1,3 +1,5 @@ +import uuid + from ansible_base.lib.utils.views.ansible_base import AnsibleBaseView from rest_framework import status from rest_framework.decorators import api_view @@ -6,6 +8,8 @@ from rest_framework.viewsets import ModelViewSet from rest_framework.viewsets import ReadOnlyModelViewSet +from core.tasks.demo import sumbit_hello_world + from .models import Automation from .models import ControllerLabel from .models import Pattern @@ -94,4 +98,9 @@ def ping(request: Request) -> Response: @api_view(["GET"]) def test(request: Request) -> Response: - return Response(data={"hello": "world"}, status=200) + text = f"hello world from uuid = {uuid.uuid4()}" + id = sumbit_hello_world(text) + return Response( + f"Task submitted (uuid={id}), check dispatcher logs. Should print '{text}'", + status=200, + ) diff --git a/pattern_service/settings/development_defaults.py b/pattern_service/settings/development_defaults.py index 7b9760e7..aadd6675 100644 --- a/pattern_service/settings/development_defaults.py +++ b/pattern_service/settings/development_defaults.py @@ -36,5 +36,6 @@ "handlers": ["console"], "level": "INFO", }, + "dispatcherd": {"handlers": ["console"], "level": "INFO"}, }, } diff --git a/pattern_service/settings/dispatcher.py b/pattern_service/settings/dispatcher.py index 07f9b489..2fd7ff31 100644 --- a/pattern_service/settings/dispatcher.py +++ b/pattern_service/settings/dispatcher.py @@ -16,60 +16,48 @@ from dynaconf import Dynaconf -def convert_to_bool(value: str) -> bool: - if isinstance(value, bool): - return value - return str(value).lower() in ("yes", "true", "1") - - def override_dispatcher_settings(loaded_settings: Dynaconf) -> None: - feature_dispatcherd = convert_to_bool(loaded_settings.get("FEATURE_DISPATCHERD")) - if feature_dispatcherd: - databases = loaded_settings.get("DATABASES", {}) - if databases and "default" not in databases: - raise ImproperlyConfigured( - "DATABASES settings must contain a 'default' key" - ) - - db_host = loaded_settings.get("DB_HOST", "127.0.0.1") - db_port = loaded_settings.get("DB_PORT", 5432) - db_user = loaded_settings.get("DB_USER", "postgres") - db_user_pass = loaded_settings.get("DB_PASSWORD") - db_name = loaded_settings.get("DB_NAME", "pattern_db") - db_app_name = loaded_settings.get("DB_APP_NAME", "dispatcher_pattern_service") - db_sslmode = loaded_settings.get("DB_SSLMODE", default="allow") - db_sslcert = loaded_settings.get("DB_SSLCERT", default="") - db_sslkey = loaded_settings.get("DB_SSLKEY", default="") - db_sslrootcert = loaded_settings.get("DB_SSLROOTCERT", default="") - - databases["default"] = { - "ENGINE": "django.db.backends.postgresql", - "HOST": db_host, - "PORT": db_port, - "USER": db_user, - "PASSWORD": db_user_pass, - "NAME": db_name, - "OPTIONS": { - "sslmode": db_sslmode, - "sslcert": db_sslcert, - "sslkey": db_sslkey, - "sslrootcert": db_sslrootcert, - }, - } - - dispatcher_conninfo = ( - f"dbname={db_name} user={db_user} password={db_user_pass} " - f"host={db_host} port={db_port} application_name={db_app_name}" - ) - dispatcher_node_id = loaded_settings.get("DISPATCHER_NODE_ID", default="") - config = loaded_settings.get("DISPATCHER_CONFIG") - config["brokers"]["pg_notify"]["config"].update( - {"conninfo": dispatcher_conninfo} - ) - if dispatcher_node_id: - config["service"]["main_kwargs"]["node_id"] = dispatcher_node_id - - loaded_settings.update( - {"DATABASES": databases, "DISPATCHER_CONFIG": config}, - loader_identifier="settings:override_dispatcher_settings", - ) + databases = loaded_settings.get("DATABASES", {}) + if databases and "default" not in databases: + raise ImproperlyConfigured("DATABASES settings must contain a 'default' key") + + db_host = loaded_settings.get("DB_HOST", "127.0.0.1") + db_port = loaded_settings.get("DB_PORT", 5432) + db_user = loaded_settings.get("DB_USER", "postgres") + db_user_pass = loaded_settings.get("DB_PASSWORD") + db_name = loaded_settings.get("DB_NAME", "pattern_db") + db_app_name = loaded_settings.get("DB_APP_NAME", "dispatcher_pattern_service") + db_sslmode = loaded_settings.get("DB_SSLMODE", default="allow") + db_sslcert = loaded_settings.get("DB_SSLCERT", default="") + db_sslkey = loaded_settings.get("DB_SSLKEY", default="") + db_sslrootcert = loaded_settings.get("DB_SSLROOTCERT", default="") + + databases["default"] = { + "ENGINE": "django.db.backends.postgresql", + "HOST": db_host, + "PORT": db_port, + "USER": db_user, + "PASSWORD": db_user_pass, + "NAME": db_name, + "OPTIONS": { + "sslmode": db_sslmode, + "sslcert": db_sslcert, + "sslkey": db_sslkey, + "sslrootcert": db_sslrootcert, + }, + } + + dispatcher_conninfo = ( + f"dbname={db_name} user={db_user} password={db_user_pass} " + f"host={db_host} port={db_port} application_name={db_app_name}" + ) + dispatcher_node_id = loaded_settings.get("DISPATCHER_NODE_ID", default="") + config = loaded_settings.get("DISPATCHER_CONFIG") + config["brokers"]["pg_notify"]["config"].update({"conninfo": dispatcher_conninfo}) + if dispatcher_node_id: + config["service"]["main_kwargs"]["node_id"] = dispatcher_node_id + + loaded_settings.update( + {"DATABASES": databases, "DISPATCHER_CONFIG": config}, + loader_identifier="settings:override_dispatcher_settings", + ) diff --git a/tools/podman/Containerfile.dev b/tools/podman/Containerfile.dev index d01cc194..7663b547 100644 --- a/tools/podman/Containerfile.dev +++ b/tools/podman/Containerfile.dev @@ -16,8 +16,6 @@ COPY manage.py . ENV PATTERN_SERVICE_MODE=development -RUN python3.11 manage.py migrate - EXPOSE 5000 CMD ["python3.11", "/app/manage.py", "runserver", "0.0.0.0:5000"] diff --git a/tools/podman/README.md b/tools/podman/README.md index fd4a5f31..abf4ecc1 100644 --- a/tools/podman/README.md +++ b/tools/podman/README.md @@ -9,8 +9,8 @@ Once you have a local copy, run the commands in the following sections from the ### Prerequisites -- [Docker](https://docs.docker.com/engine/installation/) on the host where the application will be deployed. After installing Docker, the Docker service must be started (depending on your OS, you may have to add the local user that uses Docker to the `docker` group, refer to the documentation for details) -- [Docker Compose](https://docs.docker.com/compose/install/). +- [Podman](https://podman.io/docs/installation) on the host where the application will be deployed. +- [Podman Compose](https://podman-desktop.io/docs/compose/setting-up-compose) or [python installation](https://pypi.org/project/podman-compose/). ## Starting the Development Environment @@ -23,9 +23,9 @@ Run the following to build the image: $ make compose-build ``` -The image will need to be rebuilt if there are any changes to `tools/docker/docker-compose.yaml`. +> The image will need to be rebuilt if there are any changes to `tools/podman/compose.yaml`. -Once the build completes, you will have a few new images in your local image cache. Use the `podman images` command to view them, as follows: +Once the build completes, you will have a `ansible/awx_devel` image in your local image cache. Use the `podman images` command to view it, as follows: ```bash (host)$ podman images @@ -46,6 +46,6 @@ Run the pattern-service-api, pattern-service-worker, and postgres containers. Th $ make compose-up ``` -For running docker-compose detached mode, start the containers using the following command: `$ make compose-up COMPOSE_UP_OPTS=-d` +> For running podman-compose in detached mode, start the containers using the following command: `$ make compose-up COMPOSE_UP_OPTS=-d` You can test the application from the url `http://localhost:8000/api/pattern-service/v1/test/` diff --git a/tools/podman/compose.yaml b/tools/podman/compose.yaml index 0872476e..f17d3b4f 100644 --- a/tools/podman/compose.yaml +++ b/tools/podman/compose.yaml @@ -4,7 +4,6 @@ x-environment: &common-env PATTERN_SERVICE_DB_NAME: pattern_db PATTERN_SERVICE_DB_USER: pattern PATTERN_SERVICE_DB_PASSWORD: pattern123 - PATTERN_SERVICE_FEATURE_DISPATCHERD: "true" PATTERN_SERVICE_MODE: development services: postgres: From 4c76306fbdc5effff5d3cef05684f2d388dc5315 Mon Sep 17 00:00:00 2001 From: aubin bikouo Date: Fri, 1 Aug 2025 10:44:12 +0200 Subject: [PATCH 5/8] Skip unit tests --- pattern_service/settings/dispatcher.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pattern_service/settings/dispatcher.py b/pattern_service/settings/dispatcher.py index 2fd7ff31..fd4a6bf5 100644 --- a/pattern_service/settings/dispatcher.py +++ b/pattern_service/settings/dispatcher.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from django.core.exceptions import ImproperlyConfigured from dynaconf import Dynaconf def override_dispatcher_settings(loaded_settings: Dynaconf) -> None: + if os.environ.get("PATTERN_SERVICE_MODE", "testing") == "testing": + # Skip this update while running unit tests + return None databases = loaded_settings.get("DATABASES", {}) if databases and "default" not in databases: raise ImproperlyConfigured("DATABASES settings must contain a 'default' key") From f582834cd9c3c8119dfecd54641acda5c6902582 Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Fri, 1 Aug 2025 11:18:10 -0400 Subject: [PATCH 6/8] Suggested changes for testing --- .github/workflows/tests.yml | 3 ++- Makefile | 11 +++++++++++ pattern_service/settings/dispatcher.py | 5 ----- pattern_service/settings/testing_defaults.py | 14 ++++++++++++++ tools/podman/compose-test.yaml | 7 +++++++ tools/podman/compose.yaml | 4 ++-- 6 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 tools/podman/compose-test.yaml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 817c064e..9603ebe0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest env: DJANGO_SETTINGS_MODULE: pattern_service.settings + COMPOSE_COMMAND: docker-compose steps: - name: Checkout code @@ -33,4 +34,4 @@ jobs: python -m pip install tox - name: Run unit tests - run: tox -e test + run: make test diff --git a/Makefile b/Makefile index fbbed511..83160a72 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ help: ## Show this help message # Container image # ------------------------------------- +CONTAINER_RUNTIME ?= podman COMPOSE_COMMAND ?= podman compose IMAGE_NAME ?= pattern-service IMAGE_TAG ?= latest @@ -63,3 +64,13 @@ requirements: ## Generate requirements.txt files from pyproject.toml pip-compile -o requirements/requirements.txt pyproject.toml pip-compile --extra dev --extra test -o requirements/requirements-dev.txt pyproject.toml pip-compile --extra test -o requirements/requirements-test.txt pyproject.toml + +# ------------------------------------- +# Test +# ------------------------------------- + +.PHONY: test +test: ## Run tests with a postgres database using docker-compose + $(COMPOSE_COMMAND) -f tools/podman/compose-test.yaml $(COMPOSE_OPTS) up -d + -tox -e test + $(COMPOSE_COMMAND) -f tools/podman/compose-test.yaml $(COMPOSE_OPTS) down diff --git a/pattern_service/settings/dispatcher.py b/pattern_service/settings/dispatcher.py index fd4a6bf5..2fd7ff31 100644 --- a/pattern_service/settings/dispatcher.py +++ b/pattern_service/settings/dispatcher.py @@ -12,16 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from django.core.exceptions import ImproperlyConfigured from dynaconf import Dynaconf def override_dispatcher_settings(loaded_settings: Dynaconf) -> None: - if os.environ.get("PATTERN_SERVICE_MODE", "testing") == "testing": - # Skip this update while running unit tests - return None databases = loaded_settings.get("DATABASES", {}) if databases and "default" not in databases: raise ImproperlyConfigured("DATABASES settings must contain a 'default' key") diff --git a/pattern_service/settings/testing_defaults.py b/pattern_service/settings/testing_defaults.py index 480e4b87..43dbf94a 100644 --- a/pattern_service/settings/testing_defaults.py +++ b/pattern_service/settings/testing_defaults.py @@ -3,3 +3,17 @@ ALLOWED_HOSTS = ["localhost", "pattern-service", "127.0.0.1"] BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = "insecure" + +DB_NAME = "test_pattern_db" +DB_USER = "postgres" +DB_PASSWORD = "insecure" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "localhost", + "PORT": 5432, + "PASSWORD": DB_PASSWORD, + "NAME": DB_NAME, + } +} diff --git a/tools/podman/compose-test.yaml b/tools/podman/compose-test.yaml new file mode 100644 index 00000000..7761312f --- /dev/null +++ b/tools/podman/compose-test.yaml @@ -0,0 +1,7 @@ +services: + postgres: + image: quay.io/sclorg/postgresql-15-c9s:latest + environment: + POSTGRESQL_ADMIN_PASSWORD: insecure + ports: + - '5432:5432' diff --git a/tools/podman/compose.yaml b/tools/podman/compose.yaml index f17d3b4f..0b2dec10 100644 --- a/tools/podman/compose.yaml +++ b/tools/podman/compose.yaml @@ -52,8 +52,8 @@ services: - /bin/bash - -c - >- - python3.11 /app/manage.py makemigrations core - && python3.11 /app/manage.py migrate core + python3.11 /app/manage.py makemigrations + && python3.11 /app/manage.py migrate && python3.11 /app/manage.py runserver 0.0.0.0:5000 ports: - "8000:5000" From 4db480ada80d99a008cb775910bc64185c1284d8 Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Fri, 1 Aug 2025 11:26:14 -0400 Subject: [PATCH 7/8] Update docs --- CONTRIBUTING.md | 51 ++++++++++++++++++++++++++++++++++++++++--------- pyproject.toml | 2 +- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a14d4654..4ef8a4e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,9 +11,11 @@ Hi there! We're excited to have you as a contributor. - [Clone the repo](#clone-the-repo) - [Configure Python environment](#configure-python-environment) - [Set env variables for development](#set-env-variables-for-development) + - [Configure postgres and run the dispatcher service](#configure-postgres-and-run-the-dispatcher-service) - [Configure and run the application](#configure-and-run-the-application) - [Updating dependencies](#updating-dependencies) - - [Running tests, linters, and code checks](#running-tests-linters-and-code-checks) + - [Running linters and code checks](#running-linters-and-code-checks) + - [Running tests](#running-tests) ## Things to know prior to submitting code @@ -21,7 +23,6 @@ Hi there! We're excited to have you as a contributor. - Take care to make sure no merge commits are in the submission, and use `git rebase` vs `git merge` for this reason. - If collaborating with someone else on the same branch, consider using `--force-with-lease` instead of `--force`. This will prevent you from accidentally overwriting commits pushed by someone else. For more information, see [git push docs](https://git-scm.com/docs/git-push#git-push---force-with-leaseltrefnamegt). - We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com) -- This repository uses a`pre-commit`, configuration, so ensure that you install pre-commit globally for your user, or by using pipx. ## Build and Run the Development Environment @@ -47,6 +48,8 @@ Install required python modules for development `pip install -r requirements/requirements-dev.txt` +For standalone development tools written in Python, such as `pre-commit` and `pip-tools`, we recommend using your system package manager, `pipx` tool or `pip` user install mode (`pip install --user`), in decreasing order of preference. + ### Set env variables for development Either create a .env file in the project root containing the following env variables, or export them to your shell env: @@ -55,8 +58,29 @@ Either create a .env file in the project root containing the following env varia PATTERN_SERVICE_MODE=development ``` +### Configure postgres and run the dispatcher service + +Several endpoints in the pattern service rely on asynchronous tasks that are handled by a separate running service, the dispatcher service. This uses [PostgreSQL's](https://www.postgresql.org/) `pg_notify` ability to send asyncronous tasks from the django application to the dispatcher service. For more details, see the [dispatcherd documentation](https://github.com/ansible/dispatcherd/blob/main/README.md). + +To make use of the dispatcher, you will need to ensure that both postgres and the dispatcher service are running. _The easiest way to do this is via [docker-compose](./tools/container/README.md)_, however it is also possible to do this manually as follows: + +- Install postgres locally and create a database for the service. +- Update your local .env file to reference your postgres server and database details (these can also be exported to your shell env): + +```bash +PATTERN_SERVICE_DB_NAME= +PATTERN_SERVICE_DB_USER= +PATTERN_SERVICE_DB_PASSWORD= +PATTERN_SERVICE_DB_HOST=localhost +PATTERN_SERVICE_DB_PORT="5432 (or your postgres port)" +``` + +- Run the dispatcherd service from the root pattern service directory with `python manage.py worker` + ### Configure and run the application +In a separate terminal window, run: + `python manage.py migrate && python manage.py runserver` The application can be reached in your browser at `https://localhost:8000/`. The Django admin UI is accessible at `https://localhost:8000/admin` and the available API endpoints will be listed in the 404 information at `http://localhost:8000/api/pattern-service/v1/`. @@ -67,13 +91,22 @@ Project dependencies for all environments are specified in the [pyproject.toml f To add a new dependency: -1. Add the package to the appropriate project or optional dependencies section of the pyproject.toml file, using dependency specifiers to constrain versions. -2. Update the requirements files with the command `make requirements`. This should update the relevant requirements.txt files in the project's requirements directory. +1. Ensure you have `pip-tools` installed by running either `pipx install pip-tools` or `pip install -u pip-tools`. +2. Add the package to the appropriate project or optional dependencies section of the pyproject.toml file, using dependency specifiers to constrain versions. +3. Update the requirements files with the command `make requirements`. This should update the relevant requirements.txt files in the project's requirements directory. + +## Running linters and code checks + +Linters, type checks, and other checks can all be run via `tox`. To see the available `tox` commands for this project, run `tox list`. -## Running tests, linters, and code checks +To run an individual tox command use the `-e` flag to specify the environment, for example: `tox -e lint` to run the linters. -Unit tests, linters, type checks, and other checks can all be run via `tox`. To see the available `tox` commands for this project, run `tox list`. +To run all checks, simply run `tox` with no options. -To run an individual tox command use the `-e` flag to specify the environment, for example: `tox -e test` to run tests with all supported python versions. -s -To run all tests and checks, simply run `tox` with no options. +## Running tests + +Running the tests requires a postgres connection. The easiest way to do this is with the [test compose file](./tools/podman/compose-test.yaml), and there is a `make` command to simplify starting the postgres container and running the tests: + +```bash +make test +``` diff --git a/pyproject.toml b/pyproject.toml index ff207b80..455689de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ ignore_missing_imports = true [tool.tox] requires = ["tox>=4.26"] -env_list = ["check", "format", "lint", "test", "type"] +env_list = ["check", "format", "lint", "type"] [tool.tox.env_run_base] package = "wheel" From 97b1847c82e935ba235c2996bdf87e366748e95d Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Fri, 1 Aug 2025 11:29:17 -0400 Subject: [PATCH 8/8] Update CI compose command --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9603ebe0..aa2d87cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest env: DJANGO_SETTINGS_MODULE: pattern_service.settings - COMPOSE_COMMAND: docker-compose + COMPOSE_COMMAND: docker compose steps: - name: Checkout code @@ -33,5 +33,8 @@ jobs: python -m pip install --upgrade pip python -m pip install tox + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + - name: Run unit tests run: make test