Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/test-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ jobs:
docker builder prune -af || true
docker compose -f docker-compose.yml down -v --remove-orphans || true

- name: Cleanup root-owned files (OpenSearch data, config, Langflow data)
- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
run: |
for i in 1 2 3; do
docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data" && break
docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
echo "Attempt $i failed, retrying in 5s..."
sleep 5
done || true
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,10 @@ jobs:
docker builder prune -af || true
docker compose -f docker-compose.yml down -v --remove-orphans || true

- name: Cleanup root-owned files (OpenSearch data, config, Langflow data)
- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
run: |
for i in 1 2 3; do
docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data" && break
docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
echo "Attempt $i failed, retrying in 5s..."
sleep 5
done || true
Expand Down
27 changes: 27 additions & 0 deletions Dockerfile.backend
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,44 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
openssl \
gosu \
&& rm -rf /var/lib/apt/lists/*

# Create a non-root user/group.
# UID/GID 1000 is the conventional first non-root account and
# matches what Podman's :U volume flag maps to.
RUN groupadd --gid 1000 appuser \
&& useradd --uid 1000 --gid 1000 --no-create-home appuser

WORKDIR /app

COPY --from=builder /app /app
COPY securityconfig/ ./securityconfig/
COPY cloud_securityconfig/ ./cloud_securityconfig/
COPY entrypoint.sh /entrypoint.sh

ENV VIRTUAL_ENV=/app/.venv
ENV PATH="/app/.venv/bin:$PATH"

# Pre-create every directory the app writes to at runtime so they are owned
# by appuser in the image layer. When Docker/Podman mounts a host volume over
# one of these paths the mount takes precedence, but the ownership baked here
# acts as a safe default when no volume is attached (e.g. CI, unit tests).
#
# Writable paths:
# keys/ - RSA JWT keys (private_key.pem / public_key.pem)
# data/ - connections.json
# config/ - config.yaml (ConfigManager runtime settings)
# flows/backup/ - Langflow flow backups (flows/ itself is COPY'd from builder)
# openrag-documents/ - uploaded documents staging area
RUN mkdir -p keys data config flows/backup openrag-documents \
&& chown -R appuser:appuser /app \
&& chmod +x /entrypoint.sh

# entrypoint.sh runs as root, re-chowns volume-mounted directories to appuser
# (belt-and-suspenders for Docker where :U is not supported), then execs the
# application as appuser via gosu.
EXPOSE 8000

ENTRYPOINT ["/entrypoint.sh"]
CMD ["python", "src/main.py"]
76 changes: 45 additions & 31 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ REPO ?= https://github.com/langflow-ai/langflow.git

# Auto-detect container runtime: prefer docker, fall back to podman
CONTAINER_RUNTIME := $(shell command -v docker >/dev/null 2>&1 && echo "docker" || echo "podman")

# Host UID/GID — evaluated once at parse time and used in Docker-assisted chown commands
# so that Alpine (running as root) can re-own volume directories back to the host user.
HOST_UID := $(shell id -u)
HOST_GID := $(shell id -g)
OPENRAG_IMAGE_REPOS := langflowai/openrag-backend langflowai/openrag-frontend langflowai/openrag-langflow langflowai/openrag-opensearch langflowai/openrag-dashboards langflow/langflow opensearchproject/opensearch opensearchproject/opensearch-dashboards
# Only pass --env-file if the file actually exists
ifneq (,$(wildcard $(ENV_FILE)))
Expand Down Expand Up @@ -70,6 +75,22 @@ define test_jwt_opensearch
echo "";
endef

# Fix ownership of backend volume directories
# Re-owns directories to the host user so local dev (make backend) can always read/write them,
# even after a container run chowned them to UID 1000 (appuser). Runs via Docker Alpine as root
# so it succeeds regardless of current ownership; falls back to native chown if Docker is unavailable.
# Usage: $(call fix_backend_volume_ownership)
define fix_backend_volume_ownership
$(CONTAINER_RUNTIME) run --rm \
-v "$$(pwd)/flows:/mnt/flows" \
-v "$$(pwd)/keys:/mnt/keys" \
-v "$$(pwd)/config:/mnt/config" \
-v "$$(pwd)/data:/mnt/data" \
-v "$$(pwd)/openrag-documents:/mnt/openrag-documents" \
alpine sh -c "chown -R $(HOST_UID):$(HOST_GID) /mnt/flows /mnt/keys /mnt/config /mnt/data /mnt/openrag-documents && chmod 775 /mnt/flows /mnt/keys /mnt/config /mnt/data /mnt/openrag-documents" 2>/dev/null \
|| { chown -R $(HOST_UID):$(HOST_GID) flows keys config data openrag-documents 2>/dev/null || true; chmod 775 flows keys config data openrag-documents 2>/dev/null || true; }
endef

######################
# PHONY TARGETS
######################
Expand All @@ -80,7 +101,7 @@ endef
backend frontend docling docling-stop install-be install-fe build-be build-fe build-os build-lf logs-be logs-fe logs-lf logs-os \
shell-be shell-lf shell-os restart status health db-reset clear-os-data flow-upload setup factory-reset \
dev-branch build-langflow-dev stop-dev clean-dev logs-dev logs-lf-dev shell-lf-dev restart-dev status-dev \
ensure-langflow-data
ensure-langflow-data ensure-backend-volumes

all: help

Expand Down Expand Up @@ -324,7 +345,12 @@ ensure-langflow-data: ## Create the langflow-data directory if it does not exist
@mkdir -p langflow-data
@chmod 777 langflow-data

dev: ensure-langflow-data ## Start full stack with GPU support
ensure-backend-volumes: ## Create and permission backend volume directories
@mkdir -p flows keys config data openrag-documents
@chmod 775 flows keys config data openrag-documents 2>/dev/null \
|| echo "$(YELLOW)Warning: Could not chmod backend volume directories.$(NC)"

dev: ensure-langflow-data ensure-backend-volumes ## Start full stack with GPU support
@echo "$(YELLOW)Starting OpenRAG with GPU support...$(NC)"
$(COMPOSE_CMD) -f docker-compose.yml -f docker-compose.gpu.yml up -d
@echo "$(PURPLE)Services started!$(NC)"
Expand All @@ -334,7 +360,7 @@ dev: ensure-langflow-data ## Start full stack with GPU support
@echo " $(CYAN)OpenSearch:$(NC) http://localhost:9200"
@echo " $(CYAN)Dashboards:$(NC) http://localhost:5601"

dev-cpu: ensure-langflow-data ## Start full stack with CPU only
dev-cpu: ensure-langflow-data ensure-backend-volumes ## Start full stack with CPU only
@echo "$(YELLOW)Starting OpenRAG with CPU only...$(NC)"
$(COMPOSE_CMD) up -d
@echo "$(PURPLE)Services started!$(NC)"
Expand All @@ -344,7 +370,7 @@ dev-cpu: ensure-langflow-data ## Start full stack with CPU only
@echo " $(CYAN)OpenSearch:$(NC) http://localhost:9200"
@echo " $(CYAN)Dashboards:$(NC) http://localhost:5601"

dev-local: ensure-langflow-data ## Start infrastructure for local development
dev-local: ensure-langflow-data ensure-backend-volumes ## Start infrastructure for local development
@echo "$(YELLOW)Starting infrastructure only (for local development)...$(NC)"
$(COMPOSE_CMD) -f docker-compose.yml -f docker-compose.gpu.yml up -d opensearch openrag-backend dashboards langflow
@echo "$(PURPLE)Infrastructure started!$(NC)"
Expand All @@ -355,7 +381,7 @@ dev-local: ensure-langflow-data ## Start infrastructure for local development
@echo ""
@echo "$(YELLOW)Now run 'make backend' and 'make frontend' in separate terminals$(NC)"

dev-local-cpu: ensure-langflow-data ## Start infrastructure for local development, with CPU only
dev-local-cpu: ensure-langflow-data ensure-backend-volumes ## Start infrastructure for local development, with CPU only
@echo "$(YELLOW)Starting infrastructure only (for local development)...$(NC)"
$(COMPOSE_CMD) up -d opensearch openrag-backend dashboards langflow
@echo "$(PURPLE)Infrastructure started!$(NC)"
Expand All @@ -366,7 +392,7 @@ dev-local-cpu: ensure-langflow-data ## Start infrastructure for local developmen
@echo ""
@echo "$(YELLOW)Now run 'make backend' and 'make frontend' in separate terminals$(NC)"

dev-local-build-lf: ensure-langflow-data ## Start infrastructure for local development, building only Langflow image
dev-local-build-lf: ensure-langflow-data ensure-backend-volumes ## Start infrastructure for local development, building only Langflow image
@echo "$(YELLOW)Building Langflow image...$(NC)"
$(COMPOSE_CMD) -f docker-compose.yml -f docker-compose.gpu.yml build langflow
@echo "$(YELLOW)Starting infrastructure only (for local development)...$(NC)"
Expand All @@ -379,7 +405,7 @@ dev-local-build-lf: ensure-langflow-data ## Start infrastructure for local devel
@echo ""
@echo "$(YELLOW)Now run 'make backend' and 'make frontend' in separate terminals$(NC)"

dev-local-build-lf-cpu: ensure-langflow-data ## Start infrastructure for local development, building only Langflow image with CPU only
dev-local-build-lf-cpu: ensure-langflow-data ensure-backend-volumes ## Start infrastructure for local development, building only Langflow image with CPU only
@echo "$(YELLOW)Building Langflow image (CPU)...$(NC)"
$(COMPOSE_CMD) build langflow
@echo "$(YELLOW)Starting infrastructure only (for local development)...$(NC)"
Expand All @@ -398,7 +424,7 @@ dev-local-build-lf-cpu: ensure-langflow-data ## Start infrastructure for local d
# Usage: make dev-branch BRANCH=test-openai-responses
# make dev-branch BRANCH=feature-x REPO=https://github.com/myorg/langflow.git

dev-branch: ensure-langflow-data ## Build & run full stack with custom Langflow branch
dev-branch: ensure-langflow-data ensure-backend-volumes ## Build & run full stack with custom Langflow branch
@echo "$(YELLOW)Building Langflow from branch: $(BRANCH)$(NC)"
@echo " $(CYAN)Repository:$(NC) $(REPO)"
@echo ""
Expand All @@ -414,7 +440,7 @@ dev-branch: ensure-langflow-data ## Build & run full stack with custom Langflow
@echo " $(CYAN)OpenSearch:$(NC) http://localhost:9200"
@echo " $(CYAN)Dashboards:$(NC) http://localhost:5601"

dev-branch-cpu: ensure-langflow-data ## Build & run full stack with custom Langflow branch and CPU only mode
dev-branch-cpu: ensure-langflow-data ensure-backend-volumes ## Build & run full stack with custom Langflow branch and CPU only mode
@echo "$(YELLOW)Building Langflow from branch: $(BRANCH)$(NC)"
@echo " $(CYAN)Repository:$(NC) $(REPO)"
@echo ""
Expand All @@ -441,7 +467,7 @@ stop-dev: ## Stop dev environment containers
$(COMPOSE_CMD) -f docker-compose.dev.yml down
@echo "$(PURPLE)Dev environment stopped.$(NC)"

restart-dev: ensure-langflow-data ## Restart dev environment
restart-dev: ensure-langflow-data ensure-backend-volumes ## Restart dev environment
@echo "$(YELLOW)Restarting dev environment with branch: $(BRANCH)$(NC)"
$(COMPOSE_CMD) -f docker-compose.dev.yml down
GIT_BRANCH=$(BRANCH) GIT_REPO=$(REPO) $(COMPOSE_CMD) -f docker-compose.dev.yml up -d
Expand Down Expand Up @@ -543,7 +569,8 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove image
fi; \
if [ -f "keys/private_key.pem" ] || [ -f "keys/public_key.pem" ]; then \
echo "Removing JWT keys..."; \
rm -f keys/private_key.pem keys/public_key.pem; \
rm -f keys/private_key.pem keys/public_key.pem 2>/dev/null || \
$(CONTAINER_RUNTIME) run --rm -v "$$(pwd)/keys:/keys" alpine rm -f /keys/private_key.pem /keys/public_key.pem 2>/dev/null || true; \
echo "$(PURPLE)JWT keys removed$(NC)"; \
fi; \
echo "$(YELLOW)Removing OpenRAG images...$(NC)"; \
Expand All @@ -559,6 +586,7 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove image
backend: ## Run backend locally
@echo "$(YELLOW)Starting backend locally...$(NC)"
@if [ ! -f $(ENV_FILE) ]; then echo "$(RED)$(ENV_FILE) file not found. Copy .env.example to it first$(NC)"; exit 1; fi
@$(call fix_backend_volume_ownership)
uv run python src/main.py

frontend: ## Run frontend locally
Expand Down Expand Up @@ -682,19 +710,10 @@ test-integration: ## Run integration tests (requires infrastructure)
@echo "$(YELLOW)Make sure to run 'make dev-local' first!$(NC)"
uv run pytest tests/integration/core/ -v

test-ci: ensure-langflow-data ## Start infra, run integration + SDK tests, tear down (uses DockerHub images)
@chmod 777 langflow-data
test-ci: ensure-langflow-data ensure-backend-volumes ## Start infra, run integration + SDK tests, tear down (uses DockerHub images)
@set -e; \
echo "$(YELLOW)Installing test dependencies...$(NC)"; \
uv sync --group dev; \
if [ ! -f keys/private_key.pem ]; then \
echo "$(YELLOW)Generating RSA keys for JWT signing...$(NC)"; \
uv run python -c "from src.main import generate_jwt_keys; generate_jwt_keys()"; \
else \
echo "$(CYAN)RSA keys already exist, ensuring correct permissions...$(NC)"; \
chmod 600 keys/private_key.pem 2>/dev/null || true; \
chmod 644 keys/public_key.pem 2>/dev/null || true; \
fi; \
echo "::group::Cleanup, Pull & Build Images"; \
echo "$(YELLOW)Cleaning up old containers and volumes...$(NC)"; \
$(COMPOSE_CMD) down -v 2>/dev/null || true; \
Expand Down Expand Up @@ -729,6 +748,8 @@ test-ci: ensure-langflow-data ## Start infra, run integration + SDK tests, tear
for i in $$(seq 1 60); do \
$(CONTAINER_RUNTIME) exec openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1 && break || sleep 2; \
done; \
echo "$(YELLOW)Fixing JWT key ownership for test runner (host UID $$(id -u))...$(NC)"; \
$(CONTAINER_RUNTIME) run --rm -v $$(pwd)/keys:/keys alpine sh -c "chown $$(id -u):$$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true; \
echo "$(YELLOW)Waiting for OpenSearch security config to be fully applied...$(NC)"; \
for i in $$(seq 1 60); do \
if $(CONTAINER_RUNTIME) logs os 2>&1 | grep -q "Security configuration applied successfully"; then \
Expand Down Expand Up @@ -812,19 +833,10 @@ test-ci: ensure-langflow-data ## Start infra, run integration + SDK tests, tear
$(COMPOSE_CMD) down -v 2>/dev/null || true; \
exit $$TEST_RESULT

test-ci-local: ensure-langflow-data ## Same as test-ci but builds all images locally
@chmod 777 langflow-data
test-ci-local: ensure-langflow-data ensure-backend-volumes ## Same as test-ci but builds all images locally
@set -e; \
echo "$(YELLOW)Installing test dependencies...$(NC)"; \
uv sync --group dev; \
if [ ! -f keys/private_key.pem ]; then \
echo "$(YELLOW)Generating RSA keys for JWT signing...$(NC)"; \
uv run python -c "from src.main import generate_jwt_keys; generate_jwt_keys()"; \
else \
echo "$(CYAN)RSA keys already exist, ensuring correct permissions...$(NC)"; \
chmod 600 keys/private_key.pem 2>/dev/null || true; \
chmod 644 keys/public_key.pem 2>/dev/null || true; \
fi; \
echo "::group::Cleanup & Build Images"; \
echo "$(YELLOW)Cleaning up old containers and volumes...$(NC)"; \
$(COMPOSE_CMD) down -v 2>/dev/null || true; \
Expand Down Expand Up @@ -860,6 +872,8 @@ test-ci-local: ensure-langflow-data ## Same as test-ci but builds all images loc
for i in $$(seq 1 60); do \
$(CONTAINER_RUNTIME) exec openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1 && break || sleep 2; \
done; \
echo "$(YELLOW)Fixing JWT key ownership for test runner (host UID $$(id -u))...$(NC)"; \
$(CONTAINER_RUNTIME) run --rm -v $$(pwd)/keys:/keys alpine sh -c "chown $$(id -u):$$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true; \
echo "$(YELLOW)Waiting for OpenSearch security config to be fully applied...$(NC)"; \
for i in $$(seq 1 60); do \
if $(CONTAINER_RUNTIME) logs os 2>&1 | grep -q "Security configuration applied successfully"; then \
Expand Down
23 changes: 23 additions & 0 deletions entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/sh
# entrypoint.sh — run as root, fix volume-mount ownership, then drop to appuser.
#
# When Docker (not Podman) mounts a host directory the ownership reflects the
# host filesystem, which may not be UID/GID 1000. Podman handles this with the
# :U compose flag; this script is the belt-and-suspenders fallback for Docker
# and any other runtime that does not remap UIDs automatically.
#
# Each path listed below corresponds to a volume mount in docker-compose.yml.
# The chown is a no-op when the directory is already owned by appuser (e.g.
# fresh Podman start), so there is no cost to running this unconditionally.

set -e

chown -R appuser:appuser \
/app/keys \
/app/flows \
/app/config \
/app/data \
/app/openrag-documents \
2>/dev/null || true

exec gosu appuser "$@"
Loading
Loading