diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 9ff4ef9..e49a5e4 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -15,8 +15,8 @@ services: DB_USER: pulp-manager DB_PASSWORD: pulp-manager JWT_SECRET: test_secret - PULP_MANAGER_CONFIG_PATH: /workspace/local_config.ini - PULP_SYNC_CONFIG_PATH: /workspace/local_pulp_config.yml + PULP_MANAGER_CONFIG_PATH: /workspace/demo/config.ini + PULP_SYNC_CONFIG_PATH: /workspace/demo/pulp-config.yml Is_local: "true" networks: - pulp-devcontainer-net diff --git a/.devcontainer/test_config.ini b/.devcontainer/test_config.ini new file mode 100644 index 0000000..a96c2ac --- /dev/null +++ b/.devcontainer/test_config.ini @@ -0,0 +1,50 @@ +[database] +user=pulp-manager +password=pulp-manager +host=mariadb +port=3306 +db_name=pulp_manager + +[auth] +method=ldap +use_ssl=false +ldap_servers=server:389 +base_dn=ou=people,dc=pulpproject,dc=com +default_domain=pulpproject.com +jwt_algorithm=HS256 +jwt_token_lifetime_mins=480 +admin_group=pulpmaster-rw +require_jwt_auth=false + +[pulp] +deb_signing_service=pulp_deb +banned_package_regex=bannedexample|another +internal_domains=pulp-primary,pulp-secondary +git_repo_config=https://github.com/example/repo-config.git +git_repo_config_dir=repo_config +password=password +internal_package_prefix=int_ +package_name_replacement_pattern= +package_name_replacement_rule= +remote_tls_validation=true +use_https_for_sync=false + +[redis] +host=redis-manager +port=6379 +db=0 +max_page_size=24 + +[remotes] +sock_connect_timeout=120.0 +sock_read_timeout=600.0 + +[paging] +default_page_size=50 +max_page_size=20000 + +[vault] +repo_secret_namespace=test-namespace +enabled=false +url=http://localhost:8200 +vault_addr=http://localhost:8200 \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6111977 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +demo/assets/keys/gpg/ +demo/assets/certs/ +.claude/ +.coverage +venv/ +__pycache__/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 82adb58..0790e5c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,13 @@ __pycache__ venv + +# Demo environment runtime files +demo/assets/certs/* +!demo/assets/certs/.gitkeep +demo/assets/keys/gpg/ +demo/assets/keys/*.pem +demo/assets/keys/*.key + +# Other runtime files +.coverage +.claude/ diff --git a/Dockerfile b/Dockerfile index 4e5129f..4e50084 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN mkdir -p /pulp_manager \ && groupadd pulp_manager \ && useradd -u 10001 pulp_manager -g pulp_manager -d /pulp_manager/ \ && apt-get update \ - && apt-get install -y python3-venv netcat \ + && apt-get install -y python3-venv netcat-openbsd \ && python3 -m venv /opt/venv WORKDIR /pulp_manager @@ -33,20 +33,14 @@ RUN /opt/venv/bin/pip install --upgrade pip \ FROM base as final # Install runtime dependencies including Git and make -RUN apt-get update && apt-get install -y netcat git make python3-dev libsasl2-dev libldap2-dev libssl-dev default-libmysqlclient-dev build-essential && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y netcat-openbsd git make python3-dev libsasl2-dev libldap2-dev libssl-dev default-libmysqlclient-dev build-essential && rm -rf /var/lib/apt/lists/* # Copy virtual environment from builder stage COPY --from=builder /opt/venv /opt/venv # Copy requirements file COPY --from=builder /pulp_manager/requirements.txt ./ -# Copy application code and other necessary files -COPY alembic.ini pylint.rc pytest.ini wait_db.sh *.yml ./ -ADD alembic ./alembic/. -ADD Makefile . -ADD local_config.ini . -ADD local_pulp_config.yml ./local_pulp_config.yml -ADD pulp-manager.sh . -ADD pulp_manager ./pulp_manager/. +# Copy the entire project +COPY . . # Ensure correct permissions RUN chown -R pulp_manager:pulp_manager /pulp_manager \ diff --git a/Makefile b/Makefile index 8948abe..ac99ea0 100644 --- a/Makefile +++ b/Makefile @@ -1,59 +1,62 @@ -.DEFAULT_GOAL:=all +.DEFAULT_GOAL:=h ROOT_DIR := $(dir $(lastword $(MAKEFILE_LIST))) PKG_NAME := pulp_manager -.PHONY : all -all: +.PHONY : h help +h help: @printf "%s\n" "Usage: make " @printf "\n%s\n" "Targets:" @printf " %-22s%s\n" \ + "h|help" "Print this help" \ "t|test" "Run all tests" \ "l|lint" "Run lint" \ "c|cover" "Run coverage for all tests" \ "venv" "Create virtualenv" \ - "bdist" "Create wheel file" \ "clean" "Clean workspace" \ - "h|help" "Print this help" - "run-pulp3" "Start Pulp 3 locally with Docker Compose" - - -.PHONY : h help -h help: all + "demo" "Run demo environment" .PHONY : l lint l lint: venv @echo "# pylint"; \ ./venv/bin/pylint --rcfile ./pylint.rc pulp_manager/ +check-devcontainer: + @if [ -z "$$Is_local" ] && [ -z "$$DEVCONTAINER" ]; then \ + echo "ERROR: Tests must be run in devcontainer environment!"; \ + echo ""; \ + echo "To run tests:"; \ + echo " 1. Open VS Code"; \ + echo " 2. Use Command Palette (Cmd/Ctrl+Shift+P)"; \ + echo " 3. Select 'Dev Containers: Reopen in Container'"; \ + echo " 4. Wait for container to build"; \ + echo " 5. Run: make t"; \ + echo ""; \ + exit 1; \ + fi + .PHONY : t test -t test: venv +t test: venv check-devcontainer @./venv/bin/pytest -v .PHONY : c cover -c cover: venv +c cover: venv check-devcontainer @. venv/bin/activate; \ coverage erase; \ coverage run --source=. --omit=pulp_manager/tests/unit/mock_repository.py -m pytest -v && coverage report --fail-under=90; \ coverage html +.PHONY : venv venv: requirements.txt @python3 -m venv venv @. venv/bin/activate; \ pip install --upgrade pip; \ pip install -r requirements.txt -run-pulp-manager: setup-network - @echo "Starting local Docker Compose environment..." - docker compose -f dockercompose-local.yml up --build - -.PHONY : run-pulp3 -run-pulp3: setup-network - @echo "Starting Pulp 3 locally with Docker Compose..." - docker compose -f ./dockercompose-pulp3.yml up --build - -setup-network: - @echo "Creating or verifying network..." - docker network inspect pulp-net >/dev/null 2>&1 || \ - docker network create pulp-net - @echo "Network setup completed." \ No newline at end of file +.PHONY : demo +demo: venv + @echo "Setting up demo environment..." + @. venv/bin/activate && \ + pip install -q ansible 'pulp-glue>=0.29.0' 'pulp-glue-deb>=0.3.0,<0.4' && \ + ansible-galaxy collection install pulp.squeezer 2>&1 | grep -v 'Installing' && \ + ansible-playbook -i localhost, demo/ansible/playbook.yml diff --git a/README.md b/README.md index 516e78b..afcce7f 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ The Pulp Manager application is used to coordinate common Pulp workflows and provide additional reporting capabilities about a -cluster of Pulp servers. It is designed to work with Pulp3. +cluster of Pulp servers. It is designed to work with Pulp3 servers in +a primary/secondary setup. -## About the Project +## Why Pulp Manager? -We recommend that Pulp be operated in a primary/secondary setup. There -is a single Pulp instance known as the Pulp Primary which syncs repos -from the Internet and can also have custom or internal packages -uploaded to it. Secondaries are then configured to sync these -snapshots and internal repos. +Pulp Manager provides centralized orchestration of a clustger of Pulp3 +instances and is particularly usfeful for organizations with +multi-tiered or multi-zone deployments who need coordinated syncs +between primary and secondary servers. Pulp3 doesn't provide a method to schedule the synchronisation of repos, and in some repository types (deb) may require multiple steps @@ -26,7 +26,8 @@ or Jenkins. ## Core Team -This project is maintained by G-Research. For details on our team and +This project originated at [G-Research](https://github.com/G-Research) +but is now owned by the Pulp project. For details on our team and roles, please see the [MAINTAINERS.md](MAINTAINERS.md) file. ## Documentation Index @@ -128,17 +129,22 @@ components: ## Quick Start -1. **Using DevContainers (Recommended)** +1. **For Development (running tests, exploring APIs, etc) ** ```bash - # Open in VS Code and select "Reopen in Container" - # Or use the CLI: + # Open in VS Code and select action "Dev Containers: Reopen in Container" + # Or use the Dev Container CLI: devcontainer up --workspace-folder . ``` + + From a terminal in the devcontainer, 'make t' will run the tests. + -2. **Manual Setup** +2. **For Demo cluster, use the make target to setup a complete Docker Compose environment** ```bash - make run-pulp-manager + make demo ``` + + When startup is finished, `docker ps` will show you the components, and all APIs will be listening. For detailed development setup, see the [Development Info](#development-info) section. @@ -168,6 +174,7 @@ default_domain=example.com jwt_algorithm=HS256 jwt_token_lifetime_mins=480 admin_group=pulpmaster-rw +require_jwt_auth=true [pulp] deb_signing_service=pulp_deb @@ -180,6 +187,7 @@ internal_package_prefix=corp_ package_name_replacement_pattern= package_name_replacement_rule= remote_tls_validation=true +use_https_for_sync=true [redis] host=redis @@ -220,6 +228,9 @@ Defines authentication allowed against the API - `jwt_token_lifetime_mins`: Number of minutes JWT is valid for - `admin_group`: Directory group user must be a member of to carry out priveldged actions agains the API +- `require_jwt_auth`: Boolean whether to require JWT authentication for + protected API endpoints. Set to false for local development environments + where authentication is not needed. Defaults to true ### pulp @@ -239,6 +250,8 @@ Settings to apply to all pulp servers the pulp repo config - `remote_tls_validation`: Boolean whether to require TLS validation of remote hosts +- `use_https_for_sync`: Boolean whether to use HTTPS for repository sync URLs. + Set to false for local HTTP-only development environments. Defaults to true. ### redis diff --git a/demo/ansible/playbook.yml b/demo/ansible/playbook.yml new file mode 100644 index 0000000..b18e0ee --- /dev/null +++ b/demo/ansible/playbook.yml @@ -0,0 +1,378 @@ +--- +# Ansible playbook to set up Pulp Manager and Pulp Server demo environment + +- name: Setup Pulp Demo Environment + hosts: localhost + connection: local + gather_facts: false + + vars: + pulp_primary_url: "http://localhost:8000" + pulp_secondary_url: "http://localhost:8001" + pulp_manager_url: "http://localhost:8080" + pulp_username: "admin" + pulp_password: "password" + project_dir: "{{ playbook_dir }}/../.." + package_file: "{{ playbook_dir }}/../../demo/assets/packages/hello_2.10-2_amd64.deb" + + tasks: + - name: Setup Docker network + shell: docker network inspect pulp-net >/dev/null 2>&1 || docker network create pulp-net + + - name: Create key directories + file: + path: "{{ item }}" + state: directory + mode: '0700' + loop: + - "{{ project_dir }}/demo/assets/certs" + - "{{ project_dir }}/demo/assets/keys/gpg" + - "{{ project_dir }}/demo/assets/nginx-conf" + + - name: Check if database encryption key exists + stat: + path: "{{ project_dir }}/demo/assets/certs/database_fields.symmetric.key" + register: db_key_stat + + - name: Generate database encryption key + shell: openssl rand -base64 32 > "{{ project_dir }}/demo/assets/certs/database_fields.symmetric.key" + when: not db_key_stat.stat.exists + + - name: Check if GPG public key exists + stat: + path: "{{ project_dir }}/demo/assets/keys/gpg/public.key" + register: gpg_key_stat + + - name: Check if GPG public key is empty + stat: + path: "{{ project_dir }}/demo/assets/keys/gpg/public.key" + register: gpg_key_size + when: gpg_key_stat.stat.exists + + - name: Clean GPG directory if key is missing or empty + file: + path: "{{ project_dir }}/demo/assets/keys/gpg" + state: absent + when: not gpg_key_stat.stat.exists or (gpg_key_stat.stat.exists and gpg_key_size.stat.size == 0) + + - name: Recreate GPG directory + file: + path: "{{ project_dir }}/demo/assets/keys/gpg" + state: directory + mode: '0700' + when: not gpg_key_stat.stat.exists or (gpg_key_stat.stat.exists and gpg_key_size.stat.size == 0) + + - name: Generate GPG signing keys + shell: | + cat > /tmp/gpg-batch-config </dev/null || true + GNUPGHOME={{ project_dir }}/demo/assets/keys/gpg gpg --batch --no-default-keyring --keyring {{ project_dir }}/demo/assets/keys/gpg/pubring.kbx --gen-key /tmp/gpg-batch-config + GNUPGHOME={{ project_dir }}/demo/assets/keys/gpg gpg --no-default-keyring --keyring {{ project_dir }}/demo/assets/keys/gpg/pubring.kbx --armor --export > {{ project_dir }}/demo/assets/keys/gpg/public.key + GNUPGHOME={{ project_dir }}/demo/assets/keys/gpg gpgconf --kill gpg-agent 2>/dev/null || true + rm /tmp/gpg-batch-config + when: not gpg_key_stat.stat.exists or (gpg_key_stat.stat.exists and gpg_key_size.stat.size == 0) + + - name: Start Docker Compose cluster + shell: docker compose -f demo/docker-compose.yml up -d --build + args: + chdir: "{{ project_dir }}" + + - name: Wait for Pulp Primary to be ready + pulp.squeezer.status: + pulp_url: "{{ pulp_primary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: primary_status + until: primary_status is not failed + retries: 60 + delay: 5 + + - name: Wait for Pulp Secondary to be ready + pulp.squeezer.status: + pulp_url: "{{ pulp_secondary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: secondary_status + until: secondary_status is not failed + retries: 60 + delay: 5 + + - name: Register signing service on primary + shell: | + docker exec demo-pulp-primary-1 sh -c ' + existing=$(pulpcore-manager shell -c "from pulpcore.app.models import SigningService; print(SigningService.objects.filter(name=\"deb_signing_service\").exists())" 2>/dev/null) + if [ "$existing" != "True" ]; then + key_id=$(GNUPGHOME=/opt/gpg gpg --list-secret-keys --with-colons 2>/dev/null | grep "^sec:" | cut -d: -f5 | head -1) + if [ -n "$key_id" ]; then + pulpcore-manager add-signing-service deb_signing_service /opt/scripts/deb_sign.sh "$key_id" --gnupghome /opt/gpg 2>/dev/null + echo "Signing service created on primary" + fi + else + echo "Signing service already exists on primary" + fi + ' + + - name: Register signing service on secondary + shell: | + docker exec demo-pulp-secondary-1 sh -c ' + existing=$(pulpcore-manager shell -c "from pulpcore.app.models import SigningService; print(SigningService.objects.filter(name=\"deb_signing_service\").exists())" 2>/dev/null) + if [ "$existing" != "True" ]; then + key_id=$(GNUPGHOME=/opt/gpg gpg --list-secret-keys --with-colons 2>/dev/null | grep "^sec:" | cut -d: -f5 | head -1) + if [ -n "$key_id" ]; then + pulpcore-manager add-signing-service deb_signing_service /opt/scripts/deb_sign.sh "$key_id" --gnupghome /opt/gpg 2>/dev/null + echo "Signing service created on secondary" + fi + else + echo "Signing service already exists on secondary" + fi + ' + + - name: Create int-demo-packages repository on primary + pulp.squeezer.deb_repository: + pulp_url: "{{ pulp_primary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "int-demo-packages" + description: "Internal demo repository\n base_url: int-demo-packages" + state: present + register: int_repo + + - name: Create ext-small-repo repository on primary + pulp.squeezer.deb_repository: + pulp_url: "{{ pulp_primary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "ext-small-repo" + description: "External demo repository\n base_url: ext-small-repo" + state: present + register: ext_repo + + - name: Create int-demo-packages repository on secondary + pulp.squeezer.deb_repository: + pulp_url: "{{ pulp_secondary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "int-demo-packages" + description: "Internal demo repository\n base_url: int-demo-packages" + state: present + + - name: Create ext-small-repo repository on secondary + pulp.squeezer.deb_repository: + pulp_url: "{{ pulp_secondary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "ext-small-repo" + description: "External demo repository\n base_url: ext-small-repo" + state: present + + - name: Get int-demo-packages repository details + pulp.squeezer.deb_repository: + pulp_url: "{{ pulp_primary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "int-demo-packages" + state: present + register: int_repo_info + + - name: Check if demo package already exists + uri: + url: "{{ pulp_primary_url }}{{ int_repo_info.repository.latest_version_href }}content/?limit=100" + method: GET + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + status_code: [200, 404] + register: repo_content + when: int_repo_info.repository.latest_version_href is defined and int_repo_info.repository.latest_version_href != None + + - name: Set package exists fact + set_fact: + package_exists: "{{ repo_content.json.results | selectattr('relative_path', 'defined') | selectattr('relative_path', 'search', 'hello') | list | length > 0 }}" + when: repo_content is defined and repo_content.json is defined and repo_content.status == 200 + + - name: Upload demo package + uri: + url: "{{ pulp_primary_url }}/pulp/api/v3/content/deb/packages/" + method: POST + body_format: form-multipart + body: + file: + filename: hello_2.10-2_amd64.deb + content: "{{ lookup('file', package_file) | b64encode }}" + mime_type: application/octet-stream + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + status_code: [201, 202] + register: upload_result + when: package_exists is not defined or not package_exists + + - name: Wait for upload task to complete + pulp.squeezer.task: + pulp_url: "{{ pulp_primary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + href: "{{ upload_result.json.task }}" + register: upload_task + until: upload_task.task.state == "completed" + retries: 30 + delay: 2 + when: upload_result is changed + + - name: Get content href from task + set_fact: + content_href: "{{ upload_task.task.created_resources[0] }}" + when: upload_result is changed + + - name: Add content to int-demo-packages repository + pulp.squeezer.deb_repository: + pulp_url: "{{ pulp_primary_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "int-demo-packages" + state: present + content_units: + - "{{ content_href }}" + when: upload_result is changed + + - name: Create publication for int-demo-packages + uri: + url: "{{ pulp_primary_url }}/pulp/api/v3/publications/deb/apt/" + method: POST + body_format: json + body: + repository: "{{ int_repo_info.repository.pulp_href }}" + simple: true + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + status_code: [201, 202] + register: int_publication_create + + - name: Wait for publication task + uri: + url: "{{ pulp_primary_url }}{{ int_publication_create.json.task }}" + method: GET + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + register: publication_task + until: publication_task.json.state == "completed" + retries: 30 + delay: 2 + when: int_publication_create.json.task is defined + + - name: Get publication href + set_fact: + int_publication_href: "{{ publication_task.json.created_resources[0] }}" + when: publication_task.json is defined + + - name: Check if distribution exists + uri: + url: "{{ pulp_primary_url }}/pulp/api/v3/distributions/deb/apt/?name=int-demo-packages" + method: GET + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + register: dist_check + + - name: Update existing distribution + uri: + url: "{{ pulp_primary_url }}{{ dist_check.json.results[0].pulp_href }}" + method: PATCH + body_format: json + body: + publication: "{{ int_publication_href }}" + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + status_code: [200, 202] + when: dist_check.json.count > 0 + + - name: Create new distribution + uri: + url: "{{ pulp_primary_url }}/pulp/api/v3/distributions/deb/apt/" + method: POST + body_format: json + body: + name: "int-demo-packages" + base_path: "int-demo-packages" + publication: "{{ int_publication_href }}" + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + status_code: [201, 202] + when: dist_check.json.count == 0 + + - name: Trigger repository discovery on primary server + uri: + url: "{{ pulp_manager_url }}/v1/pulp_servers/1/sync_repos" + method: POST + body_format: json + body: + max_runtime: "3600" + max_concurrent_syncs: 5 + status_code: [200, 201] + register: primary_sync + + - name: Wait for primary server repository discovery + uri: + url: "{{ pulp_manager_url }}/v1/tasks/{{ primary_sync.json.id }}" + method: GET + register: primary_sync_task + until: primary_sync_task.json.state in ["completed", "failed"] + retries: 30 + delay: 2 + + - name: Trigger repository discovery on secondary server + uri: + url: "{{ pulp_manager_url }}/v1/pulp_servers/2/sync_repos" + method: POST + body_format: json + body: + max_runtime: "3600" + max_concurrent_syncs: 5 + status_code: [200, 201] + register: secondary_sync + + - name: Wait for secondary server repository discovery + uri: + url: "{{ pulp_manager_url }}/v1/tasks/{{ secondary_sync.json.id }}" + method: GET + register: secondary_sync_task + until: secondary_sync_task.json.state in ["completed", "failed"] + retries: 30 + delay: 2 + + - name: Display completion message + debug: + msg: + - "" + - "============================================" + - "Demo Setup Complete!" + - "============================================" + - "" + - "Available repositories on primary:" + - " - ext-small-repo: {{ pulp_primary_url }}/pulp/content/ext-small-repo/" + - " - int-demo-packages: {{ pulp_primary_url }}/pulp/content/int-demo-packages/" + - "" + - "Available repositories on secondary:" + - " - ext-small-repo: {{ pulp_secondary_url }}/pulp/content/ext-small-repo/" + - " - int-demo-packages: {{ pulp_secondary_url }}/pulp/content/int-demo-packages/" + - "" + - "Pulp Manager API: {{ pulp_manager_url }}" + - "RQ Dashboard: http://localhost:9181" + - "" + - "To sync repositories from primary to secondary:" + - " curl -X POST '{{ pulp_manager_url }}/v1/pulp_servers/2/sync_repos' \\" + - " -H 'Content-Type: application/json' \\" + - " -d '{\"max_runtime\": \"3600\", \"max_concurrent_syncs\": 2, \"regex_include\": \"^int\", \"source_pulp_server_name\": \"pulp-primary:80\"}'" + - "" diff --git a/demo/assets/certs/.gitkeep b/demo/assets/certs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/demo/assets/keys/.gitkeep b/demo/assets/keys/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/demo/assets/packages/hello_2.10-2_amd64.deb b/demo/assets/packages/hello_2.10-2_amd64.deb new file mode 100644 index 0000000..f486c41 Binary files /dev/null and b/demo/assets/packages/hello_2.10-2_amd64.deb differ diff --git a/demo/assets/scripts/deb_sign.sh b/demo/assets/scripts/deb_sign.sh new file mode 100755 index 0000000..909274f --- /dev/null +++ b/demo/assets/scripts/deb_sign.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Deb signing script that calls the signing service +# $1 = input file, $2 = output file + +# Check if input file exists +if [ ! -f "$1" ]; then + echo '{"error": "Input file not found", "status": "failed"}' + exit 1 +fi + +# Set default output file if not provided +if [ -z "$2" ]; then + OUTPUT_FILE="$1.asc" +else + OUTPUT_FILE="$2" +fi + +# Call the signing service +RESPONSE=$(curl -s -F "file=@$1" http://deb-signing-service:8080/sign) + +# Extract signature content from response +SIGNATURE_CONTENT=$(echo "$RESPONSE" | python3 -c "import json, sys; data=json.load(sys.stdin); print(data.get('signature_content', ''))") +STATUS=$(echo "$RESPONSE" | python3 -c "import json, sys; data=json.load(sys.stdin); print(data.get('status', ''))") + +if [ "$STATUS" = "success" ] && [ -n "$SIGNATURE_CONTENT" ]; then + # Write signature content to output file (using printf to handle \n correctly) + printf "%s" "$SIGNATURE_CONTENT" > "$OUTPUT_FILE" + # Return JSON with the correct signature file path for Pulp + KEY_ID=$(echo "$RESPONSE" | python3 -c "import json, sys; data=json.load(sys.stdin); print(data.get('key_id', ''))") + echo "{\"signature\": \"$OUTPUT_FILE\", \"status\": \"success\", \"key_id\": \"$KEY_ID\"}" +else + # Fallback: copy input to output and return error + cp "$1" "$OUTPUT_FILE" + echo '{"error": "Signing service failed", "status": "failed"}' + exit 1 +fi \ No newline at end of file diff --git a/demo/assets/scripts/signing_service.py b/demo/assets/scripts/signing_service.py new file mode 100755 index 0000000..aaeb4f7 --- /dev/null +++ b/demo/assets/scripts/signing_service.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +import os +import json +import tempfile +import subprocess +from flask import Flask, request, jsonify +from pathlib import Path + +app = Flask(__name__) + +# Initialize GPG on startup +def init_gpg(): + """Initialize GPG with mounted keyring""" + gnupg_home = "/app/gpg" + + # Set GPG home to the mounted directory + os.environ['GNUPGHOME'] = gnupg_home + + if not os.path.exists(gnupg_home): + raise Exception(f"GPG keyring directory {gnupg_home} not found. Please mount the GPG keyring.") + + # Check if key exists + result = subprocess.run(['gpg', '--list-secret-keys'], + capture_output=True, text=True) + + if result.returncode != 0 or not result.stdout.strip(): + raise Exception("No GPG secret key found in mounted keyring") + + print("Using mounted GPG keyring") + + # Get the key ID + result = subprocess.run(['gpg', '--list-secret-keys', '--with-colons'], + capture_output=True, text=True, check=True) + + for line in result.stdout.split('\n'): + if line.startswith('sec:'): + return line.split(':')[4] + + raise Exception("No GPG key found") + +@app.route('/sign', methods=['POST']) +def sign_file(): + """Sign a file and return signature information""" + try: + # Get the file from request + if 'file' not in request.files: + return jsonify({'error': 'No file provided'}), 400 + + file = request.files['file'] + if file.filename == '': + return jsonify({'error': 'No file selected'}), 400 + + # Save file to temp location + with tempfile.NamedTemporaryFile(delete=False, suffix='.deb') as temp_file: + file.save(temp_file.name) + input_file = temp_file.name + + # Create signature file + signature_file = input_file + '.asc' + + try: + # Sign the file with GPG + subprocess.run([ + 'gpg', '--detach-sign', '--armor', + '--output', signature_file, + input_file + ], check=True, capture_output=True) + + # Read the signature + with open(signature_file, 'r') as f: + signature_content = f.read() + + return jsonify({ + 'signature': signature_file, + 'signature_content': signature_content, + 'status': 'success', + 'key_id': app.config['KEY_ID'] + }) + + finally: + # Clean up temp files + if os.path.exists(input_file): + os.unlink(input_file) + + except Exception as e: + return jsonify({'error': str(e), 'status': 'failed'}), 500 + +@app.route('/health', methods=['GET']) +def health(): + """Health check endpoint""" + return jsonify({'status': 'healthy', 'key_id': app.config.get('KEY_ID', 'unknown')}) + +@app.route('/public-key', methods=['GET']) +def get_public_key(): + """Export the public key""" + try: + key_id = app.config.get('KEY_ID') + if not key_id: + return jsonify({'error': 'No key available'}), 500 + + # Export the public key + result = subprocess.run([ + 'gpg', '--armor', '--export', key_id + ], capture_output=True, text=True, check=True) + + return jsonify({ + 'key_id': key_id, + 'public_key': result.stdout, + 'status': 'success' + }) + + except Exception as e: + return jsonify({'error': str(e), 'status': 'failed'}), 500 + +if __name__ == '__main__': + print("Initializing signing service...") + try: + key_id = init_gpg() + app.config['KEY_ID'] = key_id + print(f"Signing service ready with key ID: {key_id}") + app.run(host='0.0.0.0', port=8080, debug=False) + except Exception as e: + print(f"Failed to initialize signing service: {e}") + exit(1) \ No newline at end of file diff --git a/demo/config.ini b/demo/config.ini new file mode 100644 index 0000000..4245e80 --- /dev/null +++ b/demo/config.ini @@ -0,0 +1,47 @@ +[database] +user=pulp-manager +password=pulp-manager +host=mariadb +port=3306 +db_name=pulp_manager + +[auth] +method=ldap +use_ssl=false +ldap_servers=server:389 +base_dn=ou=people,dc=pulpproject,dc=com +default_domain=pulpproject.com +jwt_algorithm=HS256 +jwt_token_lifetime_mins=480 +admin_group=pulpmaster-rw +# Disable JWT auth requirement for demo environment +require_jwt_auth=false + +[pulp] +# Signing services exist on servers and are registered via setup-demo.sh +# But we don't set deb_signing_service here to avoid Pulp Manager auto-applying it +# during syncs (each server has different hrefs for their signing services) +# deb_signing_service=deb_signing_service +banned_package_regex=bannedexample|another +internal_domains=pulp-primary,pulp-secondary +git_repo_config_dir=repo_config +password=password +internal_package_prefix=int_ +package_name_replacement_pattern= +package_name_replacement_rule= +remote_tls_validation=false +use_https_for_sync=false + +[redis] +host=redis-manager +port=6379 +db=0 +max_page_size=24 + +[remotes] +sock_connect_timeout=120.0 +sock_read_timeout=600.0 + +[paging] +default_page_size=50 +max_page_size=20000 \ No newline at end of file diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml new file mode 100644 index 0000000..e89858d --- /dev/null +++ b/demo/docker-compose.yml @@ -0,0 +1,156 @@ +services: + pulp-primary: + image: pulp/pulp:stable + ports: + - "8000:80" + environment: + - POSTGRES_PASSWORD=password + - PULP_DEFAULT_ADMIN_PASSWORD=password + volumes: + - "./settings/settings-primary.py:/etc/pulp/settings.py:z" + - "./assets/certs/database_fields.symmetric.key:/etc/pulp/database_fields.symmetric.key:z" + - "./assets/scripts:/opt/scripts:z" + - "./assets/keys/gpg:/opt/gpg:z" + devices: + - "/dev/fuse" + networks: + - pulp-net + + pulp-secondary: + image: pulp/pulp:stable + ports: + - "8001:80" + environment: + - POSTGRES_PASSWORD=password + - PULP_DEFAULT_ADMIN_PASSWORD=password + volumes: + - "./settings/settings-secondary.py:/etc/pulp/settings.py:z" + - "./assets/certs/database_fields.symmetric.key:/etc/pulp/database_fields.symmetric.key:z" + - "./assets/scripts:/opt/scripts:z" + - "./assets/keys/gpg:/opt/gpg:z" + devices: + - "/dev/fuse" + networks: + - pulp-net + + # Pulp Manager services + mariadb: + image: mariadb:11.1.2-jammy + ports: + - "3306:3306" + environment: + MARIADB_USER: pulp-manager + MARIADB_PASSWORD: pulp-manager + MARIADB_ROOT_PASSWORD: my-root-password + MARIADB_DATABASE: pulp_manager + networks: + - pulp-net + + redis-manager: + image: redis:latest + ports: + - "6379:6379" + networks: + - pulp-net + + pulp-manager-api: + build: .. + entrypoint: ["/bin/sh", "-c"] + command: ["/usr/local/bin/pulp-manager -m && /usr/local/bin/pulp-manager -a --no-ssl --dev"] + ports: + - "8080:8000" + environment: + DB_HOSTNAME: mariadb + DB_NAME: pulp_manager + DB_USER: pulp-manager + DB_PASSWORD: pulp-manager + JWT_SECRET: test_secret + PULP_MANAGER_CONFIG_PATH: /pulp_manager/demo/config.ini + PULP_SYNC_CONFIG_PATH: /pulp_manager/demo/pulp-config.yml + Is_local: true + depends_on: + - mariadb + - redis-manager + networks: + - pulp-net + + pulp-manager-worker: + build: .. + entrypoint: ["/bin/sh", "-c"] + command: ["/usr/local/bin/pulp-manager -w"] + environment: + DB_HOSTNAME: mariadb + DB_NAME: pulp_manager + DB_USER: pulp-manager + DB_PASSWORD: pulp-manager + JWT_SECRET: test_secret + PULP_MANAGER_CONFIG_PATH: /pulp_manager/demo/config.ini + PULP_SYNC_CONFIG_PATH: /pulp_manager/demo/pulp-config.yml + Is_local: true + depends_on: + - mariadb + - redis-manager + - pulp-manager-api + networks: + - pulp-net + + pulp-manager-rq-dashboard: + build: .. + entrypoint: ["/bin/sh", "-c"] + command: ["/opt/venv/bin/rq-dashboard --bind 0.0.0.0 --port 9181 --redis-url redis://redis-manager:6379"] + ports: + - "9181:9181" + environment: + DB_HOSTNAME: mariadb + DB_NAME: pulp_manager + DB_USER: pulp-manager + DB_PASSWORD: pulp-manager + JWT_SECRET: test_secret + PULP_MANAGER_CONFIG_PATH: /pulp_manager/demo/config.ini + PULP_SYNC_CONFIG_PATH: /pulp_manager/demo/pulp-config.yml + Is_local: true + depends_on: + - redis-manager + networks: + - pulp-net + + pulp-manager-scheduler: + build: .. + entrypoint: ["/bin/sh", "-c"] + command: ["/usr/local/bin/pulp-manager -s"] + environment: + DB_HOSTNAME: mariadb + DB_NAME: pulp_manager + DB_USER: pulp-manager + DB_PASSWORD: pulp-manager + JWT_SECRET: test_secret + PULP_MANAGER_CONFIG_PATH: /pulp_manager/demo/config.ini + PULP_SYNC_CONFIG_PATH: /pulp_manager/demo/pulp-config.yml + Is_local: true + depends_on: + - mariadb + - redis-manager + - pulp-manager-api + networks: + - pulp-net + + deb-signing-service: + image: python:3.11-slim + command: > + sh -c "apt-get update && apt-get install -y gnupg && + pip install flask && + python /app/signing_service.py" + volumes: + - "./assets/scripts/signing_service.py:/app/signing_service.py:z" + - "./assets/keys/gpg:/app/gpg:z" + environment: + - GNUPGHOME=/app/gpg + networks: + - pulp-net + ports: + - "8082:8080" + restart: unless-stopped + +networks: + pulp-net: + external: true \ No newline at end of file diff --git a/demo/pulp-config.yml b/demo/pulp-config.yml new file mode 100644 index 0000000..4bc5438 --- /dev/null +++ b/demo/pulp-config.yml @@ -0,0 +1,35 @@ +pulp_servers: + pulp-primary:80: + credentials: local + repo_groups: + default: + schedule: "0 * * * *" # Every hour + max_concurrent_syncs: 1 + max_runtime: "1h" + + pulp-secondary:80: + credentials: local + repo_groups: + external_repos: + schedule: "*/5 * * * *" # Every 5 minutes + max_concurrent_syncs: 2 + max_runtime: "6h" + pulp_master: pulp-primary:80 + internal_repos: + schedule: "* * * * *" # Every 1 minute + max_concurrent_syncs: 2 + max_runtime: "1h" + pulp_master: pulp-primary:80 + +credentials: + local: + username: admin + vault_service_account_mount: local-auth + +repo_groups: + default: + regex_include: ".*" # Match all repositories + external_repos: + regex_include: "^ext" # Match repositories starting with "ext" + internal_repos: + regex_include: "^int" # Match repositories starting with "int" \ No newline at end of file diff --git a/demo/settings/settings-primary.py b/demo/settings/settings-primary.py new file mode 100644 index 0000000..2d5ede7 --- /dev/null +++ b/demo/settings/settings-primary.py @@ -0,0 +1,68 @@ +CONTENT_ORIGIN = "http://pulp-primary" + +# Database settings +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'pulp', + 'USER': 'pulp', + 'PASSWORD': 'password', + 'HOST': '127.0.0.1', + 'PORT': '5432', + } +} + +# Database encryption key +DB_ENCRYPTION_KEY = "/etc/pulp/database_fields.symmetric.key" + +# Cache settings for performance +CACHE_ENABLED = True + +# Logging +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": {"format": "pulp: %(name)s:%(levelname)s: %(message)s"} + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "simple" + } + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "INFO" + } + } +} + +# Task settings +RQ_QUEUES = { + 'default': { + 'HOST': 'localhost', + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': 360, + }, +} + +# Enable package signing +PACKAGE_SIGNING_SERVICE = True + +# Token authentication +TOKEN_AUTH_DISABLED = False + +# Content settings +MEDIA_ROOT = "/var/lib/pulp/" + +# Default admin user +PULP_DEFAULT_ADMIN_PASSWORD = "password" +# Signing service configuration +SIGNING_SERVICES = { + 'deb_signing_service': { + 'SCRIPT': '/opt/scripts/deb_sign.sh', + } +} diff --git a/demo/settings/settings-secondary.py b/demo/settings/settings-secondary.py new file mode 100644 index 0000000..845b7d8 --- /dev/null +++ b/demo/settings/settings-secondary.py @@ -0,0 +1,68 @@ +CONTENT_ORIGIN = "http://pulp-secondary" + +# Database settings +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'pulp', + 'USER': 'pulp', + 'PASSWORD': 'password', + 'HOST': '127.0.0.1', + 'PORT': '5432', + } +} + +# Database encryption key +DB_ENCRYPTION_KEY = "/etc/pulp/database_fields.symmetric.key" + +# Cache settings for performance +CACHE_ENABLED = True + +# Logging +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": {"format": "pulp: %(name)s:%(levelname)s: %(message)s"} + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "simple" + } + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "INFO" + } + } +} + +# Task settings +RQ_QUEUES = { + 'default': { + 'HOST': 'localhost', + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': 360, + }, +} + +# Enable package signing +PACKAGE_SIGNING_SERVICE = True + +# Token authentication +TOKEN_AUTH_DISABLED = False + +# Content settings +MEDIA_ROOT = "/var/lib/pulp/" + +# Default admin user +PULP_DEFAULT_ADMIN_PASSWORD = "password" +# Signing service configuration +SIGNING_SERVICES = { + 'deb_signing_service': { + 'SCRIPT': '/opt/scripts/deb_sign.sh', + } +} diff --git a/demo/settings/settings.py b/demo/settings/settings.py new file mode 100644 index 0000000..22c7c37 --- /dev/null +++ b/demo/settings/settings.py @@ -0,0 +1,68 @@ +CONTENT_ORIGIN = "http://localhost" + +# Database settings +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'pulp', + 'USER': 'pulp', + 'PASSWORD': 'password', + 'HOST': '127.0.0.1', + 'PORT': '5432', + } +} + +# Database encryption key +DB_ENCRYPTION_KEY = "/etc/pulp/database_fields.symmetric.key" + +# Cache settings for performance +CACHE_ENABLED = True + +# Logging +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": {"format": "pulp: %(name)s:%(levelname)s: %(message)s"} + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "simple" + } + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "INFO" + } + } +} + +# Task settings +RQ_QUEUES = { + 'default': { + 'HOST': 'localhost', + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': 360, + }, +} + +# Enable package signing +PACKAGE_SIGNING_SERVICE = True + +# Token authentication +TOKEN_AUTH_DISABLED = False + +# Content settings +MEDIA_ROOT = "/var/lib/pulp/" + +# Default admin user +PULP_DEFAULT_ADMIN_PASSWORD = "password" +# Signing service configuration +SIGNING_SERVICES = { + 'deb_signing_service': { + 'SCRIPT': '/opt/scripts/deb_sign.sh', + } +} diff --git a/dockercompose-local.yml b/dockercompose-local.yml deleted file mode 100644 index dde10c1..0000000 --- a/dockercompose-local.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: '3.8' - -services: - mariadb: - image: mariadb:11.1.2-jammy - ports: - - "3306:3306" - environment: - MARIADB_USER: pulp-manager - MARIADB_PASSWORD: pulp-manager - MARIADB_ROOT_PASSWORD: my-root-password - MARIADB_DATABASE: pulp_manager - networks: - - pulp-net - - redis: - image: redis:latest - networks: - - pulp-net - - pulp-manager-api: - build: . - entrypoint: ["/bin/sh", "-c"] - command: ["/usr/local/bin/pulp-manager -m && /usr/local/bin/pulp-manager -a --no-ssl --dev && /usr/local/bin/pulp-manager -w"] - volumes: - - .:/pulp_manager - ports: - - "8080:8000" - environment: - DB_HOSTNAME: mariadb - DB_NAME: pulp_manager - DB_USER: pulp-manager - DB_PASSWORD: pulp-manager - JWT_SECRET: test_secret - PULP_MANAGER_CONFIG_PATH: /pulp_manager/local_config.ini - PULP_SYNC_CONFIG_PATH: /pulp_manager/local_pulp_config.yml - #PULP_MANAGER_SKIP_PARSER_CONFIG: 1 - Is_local: true - depends_on: - - mariadb - - redis - networks: - - pulp-net - - pulp-python-worker: - build: . - entrypoint: ["/bin/sh", "-c"] - command: ["/usr/local/bin/pulp-manager -w"] - volumes: - - .:/pulp_manager - environment: - DB_HOSTNAME: mariadb - DB_NAME: pulp_manager - DB_USER: pulp-manager - DB_PASSWORD: pulp-manager - JWT_SECRET: test_secret - PULP_MANAGER_CONFIG_PATH: /pulp_manager/local_config.ini - PULP_SYNC_CONFIG_PATH: /pulp_manager/local_pulp_config.yml - #PULP_MANAGER_SKIP_PARSER_CONFIG: 1 - Is_local: true - depends_on: - - mariadb - - redis - - pulp-manager-api - networks: - - pulp-net - -networks: - pulp-net: - external: true diff --git a/dockercompose-pulp3.yml b/dockercompose-pulp3.yml deleted file mode 100644 index 4a4d75b..0000000 --- a/dockercompose-pulp3.yml +++ /dev/null @@ -1,194 +0,0 @@ -version: '3' -services: - postgres: - image: "docker.io/library/postgres:13" - ports: - - "5432:5432" - environment: - POSTGRES_USER: pulp - POSTGRES_PASSWORD: password - POSTGRES_DB: pulp - POSTGRES_INITDB_ARGS: '--auth-host=scram-sha-256' - POSTGRES_HOST_AUTH_METHOD: 'scram-sha-256' - volumes: - - "pg_data:/var/lib/postgresql/data" - - "./assets/postgres/passwd:/etc/passwd:Z" - restart: always - healthcheck: - test: pg_isready -U pulp - interval: 10s - timeout: 5s - retries: 5 - networks: - - pulp-net - - redis-pulp: - image: "docker.io/library/redis:latest" - volumes: - - "redis_data:/data" - restart: always - healthcheck: - test: redis-cli ping - interval: 10s - timeout: 5s - retries: 5 - networks: - - pulp-net - - migration_service: - image: "pulp/pulp-minimal:3.49.1" - depends_on: - postgres: - condition: service_healthy - command: pulpcore-manager migrate --noinput - volumes: - - "./assets/settings.py:/etc/pulp/settings.py:z" - - "./assets/certs:/etc/pulp/certs:z" - - "pulp:/var/lib/pulp" - networks: - - pulp-net - - signing_key_service: - image: "pulp/pulp-minimal:3.49.1" - command: sh -c "add_signing_service.sh" - depends_on: - postgres: - condition: service_healthy - migration_service: - condition: service_completed_successfully - environment: - PULP_SIGNING_KEY_FINGERPRINT: '' - volumes: - - "./assets/settings.py:/etc/pulp/settings.py:z" - - "./assets/certs:/etc/pulp/certs:z" - - "pulp:/var/lib/pulp" - networks: - - pulp-net - - set_init_password_service: - image: "pulp/pulp-minimal:3.49.1" - command: set_init_password.sh - depends_on: - postgres: - condition: service_healthy - environment: - PULP_DEFAULT_ADMIN_PASSWORD: password - volumes: - - "./assets/settings.py:/etc/pulp/settings.py:z" - - "./assets/certs:/etc/pulp/certs:z" - - "pulp:/var/lib/pulp" - networks: - - pulp-net - - pulp-web: - image: "pulp/pulp-web:3.49.1" - command: ['/usr/bin/nginx.sh'] - depends_on: - pulp_api: - condition: service_healthy - pulp_content: - condition: service_healthy - ports: - - "8000:8080" - hostname: pulp_web:8080 - user: root - volumes: - - "./assets/bin/nginx.sh:/usr/bin/nginx.sh:Z" - - "./assets/nginx/nginx.conf.template:/etc/opt/rh/rh-nginx116/nginx/nginx.conf.template:Z" - restart: always - networks: - - pulp-net - - pulp_api: - image: "pulp/pulp-minimal:3.49.1" - deploy: - replicas: 2 - command: ['pulp-api'] - depends_on: - redis-pulp: - condition: service_healthy - postgres: - condition: service_healthy - migration_service: - condition: service_completed_successfully - set_init_password_service: - condition: service_completed_successfully - signing_key_service: - condition: service_completed_successfully - hostname: pulp-api - user: pulp - environment: - PULP_SETTINGS: /etc/pulp/settings.py - PULP_ALLOWED_HOSTS: '["*"]' - volumes: - - "./assets/settings.py:/etc/pulp/settings.py:z" - - "./assets/certs:/etc/pulp/certs:z" - - "pulp:/var/lib/pulp" - restart: always - healthcheck: - test: readyz.py /pulp/api/v3/status/ - interval: 10s - timeout: 5s - retries: 5 - networks: - - pulp-net - - pulp_content: - image: "pulp/pulp-minimal:3.49.1" - deploy: - replicas: 2 - command: ['pulp-content'] - depends_on: - redis-pulp: - condition: service_healthy - postgres: - condition: service_healthy - migration_service: - condition: service_completed_successfully - hostname: pulp-content - user: pulp - volumes: - - "./assets/settings.py:/etc/pulp/settings.py:z" - - "./assets/certs:/etc/pulp/certs:z" - - "pulp:/var/lib/pulp" - restart: always - healthcheck: - test: readyz.py /pulp/content/ - interval: 10s - timeout: 5s - retries: 5 - networks: - - pulp-net - - pulp_worker: - image: "pulp/pulp-minimal:3.49.1" - deploy: - replicas: 2 - command: ['pulp-worker'] - depends_on: - redis-pulp: - condition: service_healthy - postgres: - condition: service_healthy - migration_service: - condition: service_completed_successfully - user: pulp - volumes: - - "./assets/settings.py:/etc/pulp/settings.py:z" - - "./assets/certs:/etc/pulp/certs:z" - - "pulp:/var/lib/pulp" - restart: always - networks: - - pulp-net - -volumes: - pulp: - name: pulp${DEV_VOLUME_SUFFIX:-dev} - pg_data: - name: pg_data${DEV_VOLUME_SUFFIX:-dev} - redis_data: - name: redis_data${DEV_VOLUME_SUFFIX:-dev} - -networks: - pulp-net: - external: true diff --git a/dockercompose-tests.yml b/dockercompose-tests.yml deleted file mode 100644 index fcafeb0..0000000 --- a/dockercompose-tests.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: "3.8" - -services: - webapp: - build: - context: ./ - dockerfile: Dockerfile - command: sh -c "./wait_db.sh && coverage erase && make test && make cover" - restart: "no" - environment: - DB_HOSTNAME: db - DB_NAME: pulp_manager - DB_USER: pulp-manager - DB_PASSWORD: pulp-manager - PULP_MANAGER_SKIP_PARSER_CONFIG: 1 - PULP_MANAGER_CONFIG_PATH: /pulp_manager/local_config.ini - depends_on: - - db - - db: - image: mariadb:11.1.2-jammy - restart: "no" - environment: - MARIADB_USER: pulp-manager - MARIADB_PASSWORD: pulp-manager - MARIADB_ROOT_PASSWORD: my-root-password - MARIADB_DATABASE: pulp_manager - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] - interval: 30s - timeout: 10s - retries: 3 diff --git a/local_config.ini b/local_config.ini deleted file mode 100644 index bce7054..0000000 --- a/local_config.ini +++ /dev/null @@ -1,43 +0,0 @@ -;[ca] -;root_ca_file_path="" - -[auth] -method=ldap -use_ssl=true -ldap_servers=dc.example.com -base_dn=DC=example,DC=com -default_domain=example.com -jwt_algorithm=HS256 -jwt_token_lifetime_mins=480 -admin_group=pulpmaster-rw - -[pulp] -deb_signing_service=pulp_deb -banned_package_regex=bannedexample|another -internal_domains=example.com -git_repo_config=https://git.example.com/Pulp-Repo-Config -git_repo_config_dir=repo_config -password=password -internal_package_prefix=corp_ -package_name_replacement_pattern= -package_name_replacement_rule= -;may need to be false if using http proxy to remote -remote_tls_validation=true - -[redis] -host=redis -port=6379 -db=0 -max_page_size=24 - -[remotes] -sock_connect_timeout=120.0 -sock_read_timeout=600.0 - -[paging] -default_page_size=50 -max_page_size=20000 - -[vault] -vault_addr=http://127.0.0.1:8200 -repo_secret_namespace=cle-secrets-common-dev \ No newline at end of file diff --git a/local_pulp_config.yml b/local_pulp_config.yml deleted file mode 100644 index 0243892..0000000 --- a/local_pulp_config.yml +++ /dev/null @@ -1,23 +0,0 @@ -pulp_servers: - pulp-web:8080: - credentials: local - repo_config_registration: - schedule: "0,15,30,45 * * * *" - max_runtime: "20m" - repo_groups: - external_repos: - schedule: "00 13 * * *" - max_concurrent_syncs: 2 - max_runtime: "6h" - snapshot_support: - max_concurrent_snapshots: 2 - - -credentials: - local: - username: admin - vault_service_account_mount: service-accounts # optional: Password is loaded from ini file. - -repo_groups: - external_repos: - regex_include: "^ext" diff --git a/pulp_manager/app/repositories/table_repository.py b/pulp_manager/app/repositories/table_repository.py index 1488e3f..aab373f 100644 --- a/pulp_manager/app/repositories/table_repository.py +++ b/pulp_manager/app/repositories/table_repository.py @@ -660,10 +660,17 @@ def bulk_add(self, entities: List): :return: list """ - result = self.db.scalars( - insert(self.__model__).returning(self.__model__), entities - ) - return result.all() + # Create model instances from dictionaries + new_entities = [] + for entity_dict in entities: + new_entity = self.__model__(**entity_dict) + self.db.add(new_entity) + new_entities.append(new_entity) + + # Flush to get the IDs assigned + self.db.flush() + + return new_entities def update(self, entity, **kwargs): """Updates existing entity in db but does not commit diff --git a/pulp_manager/app/routers/v1/pulp_servers.py b/pulp_manager/app/routers/v1/pulp_servers.py index f33e530..0624ad9 100644 --- a/pulp_manager/app/routers/v1/pulp_servers.py +++ b/pulp_manager/app/routers/v1/pulp_servers.py @@ -33,6 +33,19 @@ from pulp_manager.app.services import PulpManager +# Helper function to conditionally apply JWT authentication +def get_jwt_dependencies(): + """Returns JWT dependencies if required by configuration, otherwise empty list""" + if "auth" in CONFIG and "require_jwt_auth" in CONFIG["auth"]: + require_jwt = CONFIG["auth"]["require_jwt_auth"] + if isinstance(require_jwt, str): + require_jwt = require_jwt.lower() == "true" + if not require_jwt: + return [] + # Default to requiring JWT auth (backward compatible) + return [Depends(JWTBearer(allowed_groups=CONFIG["auth"]["admin_group"].split(",")))] + + pulp_server_v1_router = APIRouter( prefix="/v1/pulp_servers", tags=["pulp_servers"], @@ -329,9 +342,7 @@ def get_repo_groups_by_group_id( name="pulp_servers_v1:snapshot_repos", response_model=Task, status_code=201, - dependencies=[ - Depends(JWTBearer(allowed_groups=CONFIG["auth"]["admin_group"].split(","))) - ], + dependencies=get_jwt_dependencies(), ) def snapshot_repos( id: int, snapshot_config: PulpServerSnapshotConfig, db: get_session = Depends() @@ -357,9 +368,7 @@ def snapshot_repos( name="pulp_servers_v1:sync_repos", response_model=Task, status_code=201, - dependencies=[ - Depends(JWTBearer(allowed_groups=CONFIG["auth"]["admin_group"].split(","))) - ], + dependencies=get_jwt_dependencies(), ) def sync_repos(id: int, sync_config: PulpServerSyncConfig, db: get_session = Depends()): """Queues a repo sync job against the specified pulp server""" @@ -385,9 +394,7 @@ def sync_repos(id: int, sync_config: PulpServerSyncConfig, db: get_session = Dep name="pulp_servers_v1:remove_repos", response_model=Task, status_code=201, - dependencies=[ - Depends(JWTBearer(allowed_groups=CONFIG["auth"]["admin_group"].split(","))) - ], + dependencies=get_jwt_dependencies(), ) def remove_repos( id: int, removal_config: PulpServerRepoRemovalConfig, db: get_session = Depends() diff --git a/pulp_manager/app/services/pulp_manager.py b/pulp_manager/app/services/pulp_manager.py index 889d7ab..7cca788 100644 --- a/pulp_manager/app/services/pulp_manager.py +++ b/pulp_manager/app/services/pulp_manager.py @@ -1023,7 +1023,14 @@ def _generate_feed_from_distribution( :return: str """ - return f"https://{pulp_server_name}/pulp/content/{distribution.base_path}" + protocol = "https" + if "pulp" in CONFIG and "use_https_for_sync" in CONFIG["pulp"]: + use_https = CONFIG["pulp"]["use_https_for_sync"] + if isinstance(use_https, str): + use_https = use_https.lower() == "true" + protocol = "https" if use_https else "http" + + return f"{protocol}://{pulp_server_name}/pulp/content/{distribution.base_path}" def _get_repo_file_list_from_url(self, url: str): """Returns a list of files/directories that exist at the given url. When pulp is hosting @@ -1079,11 +1086,21 @@ def _get_apt_distributions_from_url(self, url: str): # http://pulp3mast1.example.com:24816/pulp/content/ubuntu-20.04-x86_64/focal-backports/ # need to strip 24816 from this. In the future we that logic can be removed if can make # pulp not include this - url = url.replace("http://", "https://") + + # Only convert to HTTPS if configured to do so + use_https_for_sync = True # Default to HTTPS + if "pulp" in CONFIG and "use_https_for_sync" in CONFIG["pulp"]: + use_https = CONFIG["pulp"]["use_https_for_sync"] + if isinstance(use_https, str): + use_https = use_https.lower() == "true" + use_https_for_sync = use_https + + if use_https_for_sync: + url = url.replace("http://", "https://") url = url.replace(":24816", "") if "dists/" not in url: - url = url + "dists/" + url = url.rstrip('/') + "/dists/" # This check is due to _get_apt_distributions_from_url strips off the trailing # / when getting the list of possible distributions, when called recursivley, diff --git a/pulp_manager/tests/unit/services/test_pulp_manager.py b/pulp_manager/tests/unit/services/test_pulp_manager.py index ff7dca3..c9feece 100644 --- a/pulp_manager/tests/unit/services/test_pulp_manager.py +++ b/pulp_manager/tests/unit/services/test_pulp_manager.py @@ -607,7 +607,7 @@ def test_generate_feed_from_distribution(self): "base_path": "el7-x86_64/ext-rpm" }) - expected = "https://pulp_server.domain.local/pulp/content/el7-x86_64/ext-rpm" + expected = "http://pulp_server.domain.local/pulp/content/el7-x86_64/ext-rpm" seen = self.pulp_manager._generate_feed_from_distribution("pulp_server.domain.local", distribution) assert seen == expected diff --git a/pulp_manager/tests/unit/services/test_repo_syncher.py b/pulp_manager/tests/unit/services/test_repo_syncher.py index 67cd293..39671d6 100644 --- a/pulp_manager/tests/unit/services/test_repo_syncher.py +++ b/pulp_manager/tests/unit/services/test_repo_syncher.py @@ -455,7 +455,7 @@ def test_start_remove_banned_packages_skip2(self, mock_get_remote, mock_get_repo mock_get_remote.return_value = RpmRemote(**{ "pulp_href": "/pulp/api/v3/remotes/rpm/rpm/123", "name": "test-rpm", - "url": "https://pulp.example.com/", + "url": "https://pulp-primary/", "policy": "immediate" }) diff --git a/pytest.ini b/pytest.ini index 5a6e0d7..daa5d7c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,5 +6,5 @@ norecursedirs = pulp3_bindings hashi_vault_client .git .tox dist build *.egg ven env = JWT_SECRET=test_secret PULP_MANAGER_SKIP_PARSER_CONFIG=1 - PULP_MANAGER_CONFIG_PATH=./local_config.ini - PULP_SYNC_CONFIG_PATH=./local_pulp_config.yml + PULP_MANAGER_CONFIG_PATH=./.devcontainer/test_config.ini + PULP_SYNC_CONFIG_PATH=./demo/pulp-config.yml diff --git a/requirements.txt b/requirements.txt index 16390de..4f7ef18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ dill==0.3.7 docker==7.0.0 exceptiongroup==1.1.3 fakeredis==2.20.0 -fastapi==0.104.0 +fastapi==0.104.1 freezegun==1.2.2 gitdb==4.0.11 GitPython==3.1.40 @@ -39,7 +39,7 @@ prometheus-client==0.19.0 pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 -pydantic==1.9.2 +pydantic==1.10.13 pyhcl==0.4.5 PyJWT==2.8.0 pylint==3.0.0