Set up docker with dispatcherd for local development

.dockerignore

@@ -0,0 +1,83 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Virtual environment
+.venv/
+venv/
+env/
+
+# IDE and Editor files
+.vscode/
+.idea/
+*.swp
+*.bak
+*.swo
+
+# Gitignore and other version control files
+.git/
+.gitignore
+.gitmodules
+
+# Environment variables
+.env
+.env.*
+*.env
+
+# Logs and temporary files
+*.log
+tmp/
+temp/
+
+# Database files
+*.sqlite3
+*.db
+
+# Testing
+.cache
+.coverage
+.tox
+coverage.xml
+htmlcov
+pep8.txt
+scratch
+testem.log
+.pytest_cache/
+
+# Mac OS X
+*.DS_Store
+
+# VSCode
+.vscode/

.github/workflows/tests.yml

@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DJANGO_SETTINGS_MODULE: pattern_service.settings
+      COMPOSE_COMMAND: docker compose
 
     steps:
       - name: Checkout code
@@ -32,5 +33,8 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install tox
 
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@v1
+
       - name: Run unit tests
-        run: tox -e test
+        run: make test

CONTRIBUTING.md

@@ -11,17 +11,18 @@ Hi there! We're excited to have you as a contributor.
     - [Clone the repo](#clone-the-repo)
     - [Configure Python environment](#configure-python-environment)
     - [Set env variables for development](#set-env-variables-for-development)
+    - [Configure postgres and run the dispatcher service](#configure-postgres-and-run-the-dispatcher-service)
     - [Configure and run the application](#configure-and-run-the-application)
   - [Updating dependencies](#updating-dependencies)
-  - [Running tests, linters, and code checks](#running-tests-linters-and-code-checks)
+  - [Running linters and code checks](#running-linters-and-code-checks)
+  - [Running tests](#running-tests)
 
 ## Things to know prior to submitting code
 
 - All code submissions are done through pull requests against the `main` branch.
 - Take care to make sure no merge commits are in the submission, and use `git rebase` vs `git merge` for this reason.
 - If collaborating with someone else on the same branch, consider using `--force-with-lease` instead of `--force`. This will prevent you from accidentally overwriting commits pushed by someone else. For more information, see [git push docs](https://git-scm.com/docs/git-push#git-push---force-with-leaseltrefnamegt).
 - We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com)
-- This repository uses a`pre-commit`, configuration, so ensure that you install pre-commit globally for your user, or by using pipx.
 
 ## Build and Run the Development Environment
 
@@ -47,6 +48,8 @@ Install required python modules for development
 
 `pip install -r requirements/requirements-dev.txt`
 
+For standalone development tools written in Python, such as `pre-commit` and `pip-tools`, we recommend using your system package manager, `pipx` tool or `pip` user install mode (`pip install --user`), in decreasing order of preference.
+
 ### Set env variables for development
 
 Either create a .env file in the project root containing the following env variables, or export them to your shell env:
@@ -55,8 +58,29 @@ Either create a .env file in the project root containing the following env varia
 PATTERN_SERVICE_MODE=development
 ```
 
+### Configure postgres and run the dispatcher service
+
+Several endpoints in the pattern service rely on asynchronous tasks that are handled by a separate running service, the dispatcher service. This uses [PostgreSQL's](https://www.postgresql.org/) `pg_notify` ability to send asyncronous tasks from the django application to the dispatcher service. For more details, see the [dispatcherd documentation](https://github.com/ansible/dispatcherd/blob/main/README.md).
+
+To make use of the dispatcher, you will need to ensure that both postgres and the dispatcher service are running. _The easiest way to do this is via [docker-compose](./tools/container/README.md)_, however it is also possible to do this manually as follows:
+
+- Install postgres locally and create a database for the service.
+- Update your local .env file to reference your postgres server and database details (these can also be exported to your shell env):
+
+```bash
+PATTERN_SERVICE_DB_NAME=<your database name>
+PATTERN_SERVICE_DB_USER=<your database user>
+PATTERN_SERVICE_DB_PASSWORD=<your database user password>
+PATTERN_SERVICE_DB_HOST=localhost
+PATTERN_SERVICE_DB_PORT="5432 (or your postgres port)"
+```
+
+- Run the dispatcherd service from the root pattern service directory with `python manage.py worker`
+
 ### Configure and run the application
 
+In a separate terminal window, run:
+
 `python manage.py migrate && python manage.py runserver`
 
 The application can be reached in your browser at `https://localhost:8000/`. The Django admin UI is accessible at `https://localhost:8000/admin` and the available API endpoints will be listed in the 404 information at `http://localhost:8000/api/pattern-service/v1/`.
@@ -67,13 +91,22 @@ Project dependencies for all environments are specified in the [pyproject.toml f
 
 To add a new dependency:
 
-1. Add the package to the appropriate project or optional dependencies section of the pyproject.toml file, using dependency specifiers to constrain versions.
-2. Update the requirements files with the command `make requirements`. This should update the relevant requirements.txt files in the project's requirements directory.
+1. Ensure you have `pip-tools` installed by running either `pipx install pip-tools` or `pip install -u pip-tools`.
+2. Add the package to the appropriate project or optional dependencies section of the pyproject.toml file, using dependency specifiers to constrain versions.
+3. Update the requirements files with the command `make requirements`. This should update the relevant requirements.txt files in the project's requirements directory.
+
+## Running linters and code checks
+
+Linters, type checks, and other checks can all be run via `tox`. To see the available `tox` commands for this project, run `tox list`.
 
-## Running tests, linters, and code checks
+To run an individual tox command use the `-e` flag to specify the environment, for example: `tox -e lint` to run the linters.
 
-Unit tests, linters, type checks, and other checks can all be run via `tox`. To see the available `tox` commands for this project, run `tox list`.
+To run all checks, simply run `tox` with no options.
 
-To run an individual tox command use the `-e` flag to specify the environment, for example: `tox -e test` to run tests with all supported python versions.
-s
-To run all tests and checks, simply run `tox` with no options.
+## Running tests
+
+Running the tests requires a postgres connection. The easiest way to do this is with the [test compose file](./tools/podman/compose-test.yaml), and there is a `make` command to simplify starting the postgres container and running the tests:
+
+```bash
+make test
+```

Makefile

@@ -12,16 +12,19 @@ help: ## Show this help message
 # -------------------------------------
 
 CONTAINER_RUNTIME ?= podman
+COMPOSE_COMMAND ?= podman compose
 IMAGE_NAME ?= pattern-service
 IMAGE_TAG ?= latest
+QUAY_NAMESPACE ?= ansible
+BUILD_ARGS ?= --arch amd64
 
 ensure-namespace:
 	@test -n "$$QUAY_NAMESPACE" || (echo "Error: QUAY_NAMESPACE is required to push quay.io" && exit 1)
 
 .PHONY: build
 build: ## Build the container image
 	@echo "Building container image..."
-	$(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f Dockerfile.dev --arch amd64 .
+	$(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f tools/podman/Containerfile.dev $(BUILD_ARGS) .
 
 .PHONY: clean
 clean: ## Remove container image
@@ -34,6 +37,24 @@ push: ensure-namespace build ## Tag and push container image to Quay.io
 	$(CONTAINER_RUNTIME) tag $(IMAGE_NAME):$(IMAGE_TAG) quay.io/$(QUAY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG)
 	$(CONTAINER_RUNTIME) push quay.io/$(QUAY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG)
 
+#--------------------------------------
+# Compose
+# -------------------------------------
+.PHONY: compose-build
+compose-build: ## Build the containers images for the services
+	$(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) build
+
+.PHONY: compose-up ## Build and start the containers for the services
+compose-up:
+	$(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
+
+.PHONY: compose-down
+compose-down: ## Stop containers and remove containers, network, images and volumes created by compose-up
+	$(COMPOSE_COMMAND) -f tools/podman/compose.yaml $(COMPOSE_OPTS) down --remove-orphans
+
+.PHONY: compose-restart
+compose-restart: compose-down compose-up ## Stop and remove existing infrastructure and start a new one
+
 # -------------------------------------
 # Dependencies
 # -------------------------------------
@@ -43,3 +64,13 @@ requirements: ## Generate requirements.txt files from pyproject.toml
 	pip-compile -o requirements/requirements.txt pyproject.toml
 	pip-compile --extra dev --extra test -o requirements/requirements-dev.txt pyproject.toml
 	pip-compile --extra test -o requirements/requirements-test.txt pyproject.toml
+
+# -------------------------------------
+# Test
+# -------------------------------------
+
+.PHONY: test
+test: ## Run tests with a postgres database using docker-compose
+	$(COMPOSE_COMMAND) -f tools/podman/compose-test.yaml $(COMPOSE_OPTS) up -d
+	-tox -e test
+	$(COMPOSE_COMMAND) -f tools/podman/compose-test.yaml $(COMPOSE_OPTS) down

core/apps.py

@@ -1,6 +1,16 @@
+import logging
+
+from dispatcherd.config import setup as dispatcher_setup
 from django.apps import AppConfig
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
 
 
 class CoreConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "core"
+
+    def ready(self) -> None:
+        # Configure dispatcher
+        dispatcher_setup(config=settings.DISPATCHER_CONFIG)

core/management/commands/worker.py

@@ -0,0 +1,14 @@
+import logging
+
+from dispatcherd import run_service
+from django.core.management.base import BaseCommand
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Wrapper for worker command."""
+
+    def handle(self, *args: tuple, **options: dict) -> None:
+        logger.info("Starting Pattern service dispatcherd worker.")
+        run_service()

core/tasks/demo.py

@@ -0,0 +1,18 @@
+from dispatcherd.publish import submit_task
+from dispatcherd.publish import task
+
+DISPATCHERD_DEFAULT_CHANNEL = "pattern-service-tasks"
+
+
+@task(queue=DISPATCHERD_DEFAULT_CHANNEL, decorate=False)
+def print_text(text: str) -> None:
+    print(text)
+
+
+def sumbit_hello_world(text: str):  # type: ignore
+    job_data, queue = submit_task(
+        print_text,
+        queue=DISPATCHERD_DEFAULT_CHANNEL,
+        args=(text,),
+    )
+    return job_data["uuid"]

core/tasks/hazmat.py

@@ -0,0 +1,19 @@
+import django
+from django.core.cache import cache
+from django.db import connection
+
+"""This module is an optimization for dispatcherd workers
+This sets up Django pre-fork, which must be implemented as a module to run
+on-import for compatibility with multiprocessing forkserver.
+This should never be imported by other modules, which is why it is called
+hazmat.
+"""
+
+
+django.setup()
+
+# connections may or may not be open, but
+# before forking, all connections should be closed
+
+cache.close()
+connection.close()

core/views.py

@@ -1,3 +1,5 @@
+import uuid
+
 from ansible_base.lib.utils.views.ansible_base import AnsibleBaseView
 from rest_framework import status
 from rest_framework.decorators import api_view
@@ -6,6 +8,8 @@
 from rest_framework.viewsets import ModelViewSet
 from rest_framework.viewsets import ReadOnlyModelViewSet
 
+from core.tasks.demo import sumbit_hello_world
+
 from .models import Automation
 from .models import ControllerLabel
 from .models import Pattern
@@ -94,4 +98,9 @@ def ping(request: Request) -> Response:
 
 @api_view(["GET"])
 def test(request: Request) -> Response:
-    return Response(data={"hello": "world"}, status=200)
+    text = f"hello world from uuid = {uuid.uuid4()}"
+    id = sumbit_hello_world(text)
+    return Response(
+        f"Task submitted (uuid={id}), check dispatcher logs. Should print '{text}'",
+        status=200,
+    )

pattern_service/settings/__init__.py

@@ -5,6 +5,8 @@
 from ansible_base.lib.dynamic_config import load_envvars
 from ansible_base.lib.dynamic_config import load_standard_settings_files
 
+from .dispatcher import override_dispatcher_settings
+
 try:
     from dotenv import load_dotenv
 
@@ -24,4 +26,5 @@
 )
 load_standard_settings_files(DYNACONF)
 load_envvars(DYNACONF)
+override_dispatcher_settings(DYNACONF)
 export(__name__, DYNACONF)