diff --git a/Makefile b/Makefile index e029b67..82c183c 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,21 @@ ROOT_DIR := $(dir $(lastword $(MAKEFILE_LIST))) PKG_NAME := pulp_manager +.PHONY : check-devcontainer +check-devcontainer: + @if [ -z "$$Is_local" ] && [ -z "$$DEVCONTAINER" ]; then \ + echo "ERROR: Tests must be run in devcontainer environment!"; \ + echo ""; \ + echo "To run tests:"; \ + echo " 1. Open VS Code"; \ + echo " 2. Use Command Palette (Cmd/Ctrl+Shift+P)"; \ + echo " 3. Select 'Dev Containers: Reopen in Container'"; \ + echo " 4. Wait for container to build"; \ + echo " 5. Run: make t"; \ + echo ""; \ + exit 1; \ + fi + .PHONY : h help h help: @printf "%s\n" "Usage: make " @@ -13,30 +28,18 @@ h help: "l|lint" "Run lint" \ "c|cover" "Run coverage for all tests" \ "venv" "Create virtualenv" \ + "ansible" "Install ansible in venv" \ "clean" "Clean workspace" \ "run-pulp-manager" "Run Pulp Manager services for development" \ "run-pulp3" "Run Pulp 3 primary and secondary servers" \ - "demo" "Run complete demo environment" + "demo" "Run complete demo environment" \ + "demo-repo-sync" "Upload package and run sync tasks in demo env" .PHONY : l lint l lint: venv @echo "# pylint"; \ ./venv/bin/pylint --rcfile ./pylint.rc pulp_manager/ -check-devcontainer: - @if [ -z "$$Is_local" ] && [ -z "$$DEVCONTAINER" ]; then \ - echo "ERROR: Tests must be run in devcontainer environment!"; \ - echo ""; \ - echo "To run tests:"; \ - echo " 1. Open VS Code"; \ - echo " 2. Use Command Palette (Cmd/Ctrl+Shift+P)"; \ - echo " 3. Select 'Dev Containers: Reopen in Container'"; \ - echo " 4. Wait for container to build"; \ - echo " 5. Run: make t"; \ - echo ""; \ - exit 1; \ - fi - .PHONY : t test t test: venv check-devcontainer @./venv/bin/pytest -v @@ -49,11 +52,16 @@ c cover: venv check-devcontainer coverage html .PHONY : venv -venv: requirements.txt +venv: @python3 -m venv venv @. venv/bin/activate; \ - pip install --upgrade pip; \ - pip install -r requirements.txt + pip install --upgrade pip -q; \ + pip install -r requirements.txt -q; + +.PHONY : ansible +ansible: venv + @. venv/bin/activate && \ + pip install -q ansible 'pulp-glue>=0.29.0' 'pulp-glue-deb>=0.3.0,<0.4' .PHONY : run-pulp-manager run-pulp-manager: @@ -77,9 +85,14 @@ run-pulp3: @echo "Secondary: http://localhost:8001" .PHONY : demo -demo: venv +demo: venv ansible @echo "Setting up demo environment..." @. venv/bin/activate && \ - pip install -q ansible 'pulp-glue>=0.29.0' 'pulp-glue-deb>=0.3.0,<0.4' && \ - ansible-galaxy collection install pulp.squeezer 2>&1 | grep -v 'Installing' && \ - ansible-playbook -i localhost, demo/ansible/playbook.yml + ansible-playbook -i localhost demo/ansible/playbook.yml + +.PHONY : demo-repo-sync +demo-repo-sync: venv ansible + @echo "Setting up demo environment..." + @. venv/bin/activate && \ + ansible-playbook -i localhost demo/ansible/setup_repos.yml + diff --git a/README.md b/README.md index 564067b..b15308e 100644 --- a/README.md +++ b/README.md @@ -143,9 +143,61 @@ components: ```bash make demo ``` - + When startup is finished, `docker ps` will show you the components, and all APIs will be listening. +### Demo Environment Details + +Once the demo is running, you'll have access to: + +**Available repositories on primary (http://localhost:8000):** +- `int-demo-packages`: http://localhost:8000/pulp/content/int-demo-packages/ (internal, no remote) +- `ext-demo-packages`: http://localhost:8000/pulp/content/ext-demo-packages/ (syncs from nginx.org) + +**Available repositories on secondary (http://localhost:8001):** +- `int-demo-packages`: http://localhost:8001/pulp/content/int-demo-packages/ (syncs from primary) +- `ext-demo-packages`: http://localhost:8001/pulp/content/ext-demo-packages/ (syncs from primary) + +**Services:** +- Pulp Manager API: http://localhost:8080 +- RQ Dashboard: http://localhost:9181 + +### Demo Usage Examples + +**Upload a package to int-demo-packages on primary:** +```bash +# Upload content +docker cp /path/to/package.deb demo-pulp-primary-1:/tmp/package.deb +docker exec demo-pulp-primary-1 pulp deb content upload --file /tmp/package.deb --repository int-demo-packages + +# Create publication +docker exec demo-pulp-primary-1 pulp deb publication create --repository int-demo-packages --simple + +# Update distribution (get publication href from above command) +docker exec demo-pulp-primary-1 pulp deb distribution update --name int-demo-packages --publication +``` + +**Sync ext-demo-packages on primary from nginx.org:** +```bash +curl -X POST 'http://localhost:8080/v1/pulp_servers/1/sync_repos' \ + -H 'Content-Type: application/json' \ + -d '{"max_runtime": "3600", "max_concurrent_syncs": 2, "regex_include": "^ext-demo"}' +``` + +**Sync int-demo-packages from primary to secondary:** +```bash +curl -X POST 'http://localhost:8080/v1/pulp_servers/2/sync_repos' \ + -H 'Content-Type: application/json' \ + -d '{"max_runtime": "3600", "max_concurrent_syncs": 2, "regex_include": "^int-demo", "source_pulp_server_name": "pulp-primary:80"}' +``` + +**Sync ext-demo-packages from primary to secondary:** +```bash +curl -X POST 'http://localhost:8080/v1/pulp_servers/2/sync_repos' \ + -H 'Content-Type: application/json' \ + -d '{"max_runtime": "3600", "max_concurrent_syncs": 2, "regex_include": "^ext-demo", "source_pulp_server_name": "pulp-primary:80"}' +``` + For detailed development setup, see the [Development Info](#development-info) section. @@ -182,6 +234,8 @@ banned_package_regex=bannedexample|another internal_domains=example.com git_repo_config=https://git.example.com/Pulp-Repo-Config git_repo_config_dir=repo_config +# Optional: Use local filesystem instead of git for repo configs (e.g., demo/dev) +# local_repo_config_dir=/path/to/local/repo-config password=password internal_repo_prefix=corp- external_repo_prefix=ext- @@ -249,6 +303,9 @@ Settings to apply to all pulp servers servers - `git_repo_config_dir`: Directory in `git_repo_config` which contains the pulp repo config +- `local_repo_config_dir`: Optional local filesystem path to repo config + directory. If set, scheduled repo registration will read configs from + this path instead of cloning from `git_repo_config`. - `internal_repo_prefix`: Prefix applied to a repo name when uploaded directly to Pulp primary (no remote URL). Leave blank or omit to retain original name. diff --git a/demo/ansible/playbook.yml b/demo/ansible/playbook.yml index b18e0ee..e28f930 100644 --- a/demo/ansible/playbook.yml +++ b/demo/ansible/playbook.yml @@ -1,19 +1,27 @@ --- # Ansible playbook to set up Pulp Manager and Pulp Server demo environment -- name: Setup Pulp Demo Environment +- name: Setup Pulp Manager Demo Environment hosts: localhost connection: local gather_facts: false vars: - pulp_primary_url: "http://localhost:8000" - pulp_secondary_url: "http://localhost:8001" pulp_manager_url: "http://localhost:8080" pulp_username: "admin" pulp_password: "password" project_dir: "{{ playbook_dir }}/../.." - package_file: "{{ playbook_dir }}/../../demo/assets/packages/hello_2.10-2_amd64.deb" + pulp_primary: + name: "primary" + url: "http://localhost:8000" + container: "demo-pulp-primary-1" + pulp_secondary: + name: "secondary" + url: "http://localhost:8001" + container: "demo-pulp-secondary-1" + pulp_servers: + - "{{ pulp_primary }}" + - "{{ pulp_secondary }}" tasks: - name: Setup Docker network @@ -85,272 +93,86 @@ args: chdir: "{{ project_dir }}" - - name: Wait for Pulp Primary to be ready - pulp.squeezer.status: - pulp_url: "{{ pulp_primary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - register: primary_status - until: primary_status is not failed + - name: Wait for Pulp servers to be ready + uri: + url: "{{ item.url }}/pulp/api/v3/status/" + method: GET + force_basic_auth: true + url_username: "{{ pulp_username }}" + url_password: "{{ pulp_password }}" + status_code: [200] + register: server_status + until: server_status is not failed retries: 60 delay: 5 + loop: "{{ pulp_servers }}" + loop_control: + label: "{{ item.name }}" - - name: Wait for Pulp Secondary to be ready - pulp.squeezer.status: - pulp_url: "{{ pulp_secondary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - register: secondary_status - until: secondary_status is not failed - retries: 60 - delay: 5 + - name: Install pulp-cli in Pulp containers + shell: | + docker exec {{ item.container }} bash -c "pip install --quiet pulp-cli pulp-cli-deb" + loop: "{{ pulp_servers }}" + loop_control: + label: "{{ item.name }}" - - name: Register signing service on primary + - name: Configure pulp-cli in containers shell: | - docker exec demo-pulp-primary-1 sh -c ' - existing=$(pulpcore-manager shell -c "from pulpcore.app.models import SigningService; print(SigningService.objects.filter(name=\"deb_signing_service\").exists())" 2>/dev/null) - if [ "$existing" != "True" ]; then - key_id=$(GNUPGHOME=/opt/gpg gpg --list-secret-keys --with-colons 2>/dev/null | grep "^sec:" | cut -d: -f5 | head -1) - if [ -n "$key_id" ]; then - pulpcore-manager add-signing-service deb_signing_service /opt/scripts/deb_sign.sh "$key_id" --gnupghome /opt/gpg 2>/dev/null - echo "Signing service created on primary" - fi - else - echo "Signing service already exists on primary" - fi - ' + docker exec {{ item.container }} bash -c " + mkdir -p /root/.config/pulp + cat > /root/.config/pulp/settings.toml </dev/null) if [ "$existing" != "True" ]; then key_id=$(GNUPGHOME=/opt/gpg gpg --list-secret-keys --with-colons 2>/dev/null | grep "^sec:" | cut -d: -f5 | head -1) if [ -n "$key_id" ]; then pulpcore-manager add-signing-service deb_signing_service /opt/scripts/deb_sign.sh "$key_id" --gnupghome /opt/gpg 2>/dev/null - echo "Signing service created on secondary" + echo "Signing service created on {{ item.name }}" fi else - echo "Signing service already exists on secondary" + echo "Signing service already exists on {{ item.name }}" fi ' + loop: "{{ pulp_servers }}" + loop_control: + label: "{{ item.name }}" - - name: Create int-demo-packages repository on primary - pulp.squeezer.deb_repository: - pulp_url: "{{ pulp_primary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "int-demo-packages" - description: "Internal demo repository\n base_url: int-demo-packages" - state: present - register: int_repo - - - name: Create ext-small-repo repository on primary - pulp.squeezer.deb_repository: - pulp_url: "{{ pulp_primary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "ext-small-repo" - description: "External demo repository\n base_url: ext-small-repo" - state: present - register: ext_repo - - - name: Create int-demo-packages repository on secondary - pulp.squeezer.deb_repository: - pulp_url: "{{ pulp_secondary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "int-demo-packages" - description: "Internal demo repository\n base_url: int-demo-packages" - state: present - - - name: Create ext-small-repo repository on secondary - pulp.squeezer.deb_repository: - pulp_url: "{{ pulp_secondary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "ext-small-repo" - description: "External demo repository\n base_url: ext-small-repo" - state: present - - - name: Get int-demo-packages repository details - pulp.squeezer.deb_repository: - pulp_url: "{{ pulp_primary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "int-demo-packages" - state: present - register: int_repo_info - - - name: Check if demo package already exists + - name: Wait for repos to be created on primary server uri: - url: "{{ pulp_primary_url }}{{ int_repo_info.repository.latest_version_href }}content/?limit=100" + url: "{{ pulp_manager_url }}/v1/pulp_servers/1/repos" method: GET - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - status_code: [200, 404] - register: repo_content - when: int_repo_info.repository.latest_version_href is defined and int_repo_info.repository.latest_version_href != None - - - name: Set package exists fact - set_fact: - package_exists: "{{ repo_content.json.results | selectattr('relative_path', 'defined') | selectattr('relative_path', 'search', 'hello') | list | length > 0 }}" - when: repo_content is defined and repo_content.json is defined and repo_content.status == 200 - - - name: Upload demo package - uri: - url: "{{ pulp_primary_url }}/pulp/api/v3/content/deb/packages/" - method: POST - body_format: form-multipart - body: - file: - filename: hello_2.10-2_amd64.deb - content: "{{ lookup('file', package_file) | b64encode }}" - mime_type: application/octet-stream - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - status_code: [201, 202] - register: upload_result - when: package_exists is not defined or not package_exists - - - name: Wait for upload task to complete - pulp.squeezer.task: - pulp_url: "{{ pulp_primary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - href: "{{ upload_result.json.task }}" - register: upload_task - until: upload_task.task.state == "completed" - retries: 30 + register: primary_repo_check + until: primary_repo_check.status == 200 and 'int-demo-packages' in (primary_repo_check.json | string) + retries: 35 delay: 2 - when: upload_result is changed - - - name: Get content href from task - set_fact: - content_href: "{{ upload_task.task.created_resources[0] }}" - when: upload_result is changed - - - name: Add content to int-demo-packages repository - pulp.squeezer.deb_repository: - pulp_url: "{{ pulp_primary_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "int-demo-packages" - state: present - content_units: - - "{{ content_href }}" - when: upload_result is changed - - name: Create publication for int-demo-packages + - name: Wait for repos to be created on secondary server uri: - url: "{{ pulp_primary_url }}/pulp/api/v3/publications/deb/apt/" - method: POST - body_format: json - body: - repository: "{{ int_repo_info.repository.pulp_href }}" - simple: true - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - status_code: [201, 202] - register: int_publication_create - - - name: Wait for publication task - uri: - url: "{{ pulp_primary_url }}{{ int_publication_create.json.task }}" - method: GET - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - register: publication_task - until: publication_task.json.state == "completed" - retries: 30 - delay: 2 - when: int_publication_create.json.task is defined - - - name: Get publication href - set_fact: - int_publication_href: "{{ publication_task.json.created_resources[0] }}" - when: publication_task.json is defined - - - name: Check if distribution exists - uri: - url: "{{ pulp_primary_url }}/pulp/api/v3/distributions/deb/apt/?name=int-demo-packages" - method: GET - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - register: dist_check - - - name: Update existing distribution - uri: - url: "{{ pulp_primary_url }}{{ dist_check.json.results[0].pulp_href }}" - method: PATCH - body_format: json - body: - publication: "{{ int_publication_href }}" - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - status_code: [200, 202] - when: dist_check.json.count > 0 - - - name: Create new distribution - uri: - url: "{{ pulp_primary_url }}/pulp/api/v3/distributions/deb/apt/" - method: POST - body_format: json - body: - name: "int-demo-packages" - base_path: "int-demo-packages" - publication: "{{ int_publication_href }}" - force_basic_auth: true - url_username: "{{ pulp_username }}" - url_password: "{{ pulp_password }}" - status_code: [201, 202] - when: dist_check.json.count == 0 - - - name: Trigger repository discovery on primary server - uri: - url: "{{ pulp_manager_url }}/v1/pulp_servers/1/sync_repos" - method: POST - body_format: json - body: - max_runtime: "3600" - max_concurrent_syncs: 5 - status_code: [200, 201] - register: primary_sync - - - name: Wait for primary server repository discovery - uri: - url: "{{ pulp_manager_url }}/v1/tasks/{{ primary_sync.json.id }}" + url: "{{ pulp_manager_url }}/v1/pulp_servers/2/repos" method: GET - register: primary_sync_task - until: primary_sync_task.json.state in ["completed", "failed"] - retries: 30 + register: secondary_repo_check + until: secondary_repo_check.status == 200 and 'int-demo-packages' in (secondary_repo_check.json | string) + retries: 35 delay: 2 - - name: Trigger repository discovery on secondary server - uri: - url: "{{ pulp_manager_url }}/v1/pulp_servers/2/sync_repos" - method: POST - body_format: json - body: - max_runtime: "3600" - max_concurrent_syncs: 5 - status_code: [200, 201] - register: secondary_sync - - - name: Wait for secondary server repository discovery - uri: - url: "{{ pulp_manager_url }}/v1/tasks/{{ secondary_sync.json.id }}" - method: GET - register: secondary_sync_task - until: secondary_sync_task.json.state in ["completed", "failed"] - retries: 30 - delay: 2 + - name: Display repo creation status + debug: + msg: + - "Primary server: Repos found" + - "Secondary server: Repos found" - name: Display completion message debug: @@ -360,19 +182,13 @@ - "Demo Setup Complete!" - "============================================" - "" - - "Available repositories on primary:" - - " - ext-small-repo: {{ pulp_primary_url }}/pulp/content/ext-small-repo/" - - " - int-demo-packages: {{ pulp_primary_url }}/pulp/content/int-demo-packages/" - - "" - - "Available repositories on secondary:" - - " - ext-small-repo: {{ pulp_secondary_url }}/pulp/content/ext-small-repo/" - - " - int-demo-packages: {{ pulp_secondary_url }}/pulp/content/int-demo-packages/" - - "" - "Pulp Manager API: {{ pulp_manager_url }}" - "RQ Dashboard: http://localhost:9181" - "" - - "To sync repositories from primary to secondary:" - - " curl -X POST '{{ pulp_manager_url }}/v1/pulp_servers/2/sync_repos' \\" - - " -H 'Content-Type: application/json' \\" - - " -d '{\"max_runtime\": \"3600\", \"max_concurrent_syncs\": 2, \"regex_include\": \"^int\", \"source_pulp_server_name\": \"pulp-primary:80\"}'" + - "Repositories have been created from demo/repo-config/" + - "" + - "Next step - sync repos and upload demo package:" + - " ansible-playbook demo/ansible/setup_repos.yml" + - "" + - "See README.md for usage examples." - "" diff --git a/demo/ansible/setup_repos.yml b/demo/ansible/setup_repos.yml new file mode 100644 index 0000000..3cc3181 --- /dev/null +++ b/demo/ansible/setup_repos.yml @@ -0,0 +1,135 @@ +--- +# Playbook to sync repositories and setup internal demo package +# Run this after the main playbook completes + +- name: Setup Demo Repositories + hosts: localhost + connection: local + gather_facts: false + + vars: + pulp_manager_url: "http://localhost:8080" + project_dir: "{{ playbook_dir }}/../.." + package_file: "{{ playbook_dir }}/../../demo/assets/packages/hello_2.10-2_amd64.deb" + pulp_primary: + name: "primary" + url: "http://localhost:8000" + container: "demo-pulp-primary-1" + + tasks: + - name: Trigger repository discovery and sync on primary server + uri: + url: "{{ pulp_manager_url }}/v1/pulp_servers/1/sync_repos" + method: POST + body_format: json + body: + max_runtime: "3600" + max_concurrent_syncs: 5 + status_code: [200, 201] + register: primary_sync + + - name: Wait for primary server repository discovery + uri: + url: "{{ pulp_manager_url }}/v1/tasks/{{ primary_sync.json.id }}" + method: GET + register: primary_sync_task + until: primary_sync_task.json.state in ["completed", "failed"] + retries: 60 + delay: 3 + + - name: Display primary sync result + debug: + msg: "Primary server sync: {{ primary_sync_task.json.state }}" + + - name: Check if int-demo-packages repository exists + shell: docker exec {{ pulp_primary.container }} pulp deb repository show --name 'int-demo-packages' + register: repo_check + failed_when: false + + - name: Fail if repository doesn't exist + fail: + msg: "Repository int-demo-packages does not exist. Run the main playbook first to create repos." + when: repo_check.rc != 0 + + - name: Upload package content to int-demo-packages + shell: | + docker cp {{ package_file }} {{ pulp_primary.container }}:/tmp/package.deb + docker exec {{ pulp_primary.container }} bash -c " + # Check if repo already has content + has_content=\$(pulp deb repository show --name 'int-demo-packages' | grep 'latest_version_href' | grep -v '/versions/0/') + if [ -z \"\$has_content\" ]; then + pulp deb content upload --file /tmp/package.deb --repository 'int-demo-packages' + echo 'Package uploaded' + else + echo 'Repository already has content, skipping upload' + fi + rm -f /tmp/package.deb + " + register: upload_result + + - name: Display upload result + debug: + msg: "{{ upload_result.stdout_lines }}" + + - name: Create publication and distribution for int-demo-packages + shell: | + docker exec {{ pulp_primary.container }} bash -c " + # Create publication + pulp deb publication create --repository 'int-demo-packages' --simple + + # Get latest publication href + pub_href=\$(pulp deb publication list --repository 'int-demo-packages' --limit 1 | grep 'Pulp href' | awk '{print \$3}') + + # Update distribution to point to publication + if pulp deb distribution show --name 'int-demo-packages' 2>/dev/null; then + pulp deb distribution update --name 'int-demo-packages' --publication \"\$pub_href\" + echo 'Distribution updated with new publication' + else + echo 'ERROR: Distribution int-demo-packages does not exist' + exit 1 + fi + " + register: publish_result + + - name: Display publish result + debug: + msg: "{{ publish_result.stdout_lines }}" + + - name: Trigger repository discovery on secondary server + uri: + url: "{{ pulp_manager_url }}/v1/pulp_servers/2/sync_repos" + method: POST + body_format: json + body: + max_runtime: "3600" + max_concurrent_syncs: 5 + status_code: [200, 201] + register: secondary_sync + + - name: Wait for secondary server repository discovery + uri: + url: "{{ pulp_manager_url }}/v1/tasks/{{ secondary_sync.json.id }}" + method: GET + register: secondary_sync_task + until: secondary_sync_task.json.state in ["completed", "failed"] + retries: 60 + delay: 3 + + - name: Display secondary sync result + debug: + msg: "Secondary server sync: {{ secondary_sync_task.json.state }}" + + - name: Display completion message + debug: + msg: + - "" + - "============================================" + - "Demo Repositories Setup Complete!" + - "============================================" + - "" + - "Primary server: Synced from external sources" + - "Internal package: hello_2.10-2_amd64.deb uploaded" + - "Secondary server: Synced from primary" + - "" + - "Repository: int-demo-packages is now available on both servers" + - "" diff --git a/demo/config.ini b/demo/config.ini index ccd0603..c6a38cb 100644 --- a/demo/config.ini +++ b/demo/config.ini @@ -18,13 +18,11 @@ admin_group=pulpmaster-rw require_jwt_auth=false [pulp] -# Signing services exist on servers and are registered via setup-demo.sh -# But we don't set deb_signing_service here to avoid Pulp Manager auto-applying it -# during syncs (each server has different hrefs for their signing services) # deb_signing_service=deb_signing_service banned_package_regex=bannedexample|another internal_domains=pulp-primary,pulp-secondary git_repo_config_dir=repo_config +local_repo_config_dir=/pulp_manager/demo/repo-config password=password internal_repo_prefix=int- external_repo_prefix=ext- diff --git a/demo/pulp-config.yml b/demo/pulp-config.yml index 4bc5438..5b2aa14 100644 --- a/demo/pulp-config.yml +++ b/demo/pulp-config.yml @@ -1,14 +1,20 @@ pulp_servers: pulp-primary:80: credentials: local + repo_config_registration: + schedule: "* * * * *" # Run immediately for demo setup + max_runtime: "300" repo_groups: default: schedule: "0 * * * *" # Every hour max_concurrent_syncs: 1 max_runtime: "1h" - + pulp-secondary:80: credentials: local + repo_config_registration: + schedule: "* * * * *" # Run immediately for demo setup + max_runtime: "300" repo_groups: external_repos: schedule: "*/5 * * * *" # Every 5 minutes diff --git a/demo/repo-config/internal/global.json b/demo/repo-config/internal/global.json new file mode 100644 index 0000000..661a45c --- /dev/null +++ b/demo/repo-config/internal/global.json @@ -0,0 +1,3 @@ +{ + "proxy": null +} diff --git a/demo/repo-config/internal/int-demo-packages.json b/demo/repo-config/internal/int-demo-packages.json new file mode 100644 index 0000000..cf8fb6f --- /dev/null +++ b/demo/repo-config/internal/int-demo-packages.json @@ -0,0 +1,11 @@ +{ + "name": "int-demo-packages", + "description": "Internal demo repository", + "owner": "pulp-primary", + "base_url": "int-demo-packages", + "content_repo_type": "deb", + "releases": ["stable"], + "architectures": ["amd64"], + "components": ["main"], + "ignore_missing_package_indices": true +} diff --git a/demo/repo-config/remote/ext-demo-packages.json b/demo/repo-config/remote/ext-demo-packages.json new file mode 100644 index 0000000..b665c45 --- /dev/null +++ b/demo/repo-config/remote/ext-demo-packages.json @@ -0,0 +1,11 @@ +{ + "name": "ext-demo-packages", + "description": "External demo packages from nginx.org", + "owner": "pulp-primary", + "base_url": "ext-demo-packages", + "content_repo_type": "deb", + "url": "http://nginx.org/packages/debian/", + "releases": "bookworm", + "components": "nginx", + "architectures": "amd64" +} diff --git a/demo/repo-config/remote/global.json b/demo/repo-config/remote/global.json new file mode 100644 index 0000000..661a45c --- /dev/null +++ b/demo/repo-config/remote/global.json @@ -0,0 +1,3 @@ +{ + "proxy": null +} diff --git a/pulp_manager/app/job_manager.py b/pulp_manager/app/job_manager.py index 963a4cc..7968eba 100644 --- a/pulp_manager/app/job_manager.py +++ b/pulp_manager/app/job_manager.py @@ -176,6 +176,9 @@ def _setup_repo_registration_scheduled_job(self, pulp_server: PulpServer): ): scheduler.cancel(job) + # Get local config dir from CONFIG if set, otherwise None (will clone from git) + local_config_dir = CONFIG["pulp"].get("local_repo_config_dir", None) + scheduler.cron( pulp_server.repo_config_registration_schedule, func=register_repos, @@ -184,6 +187,7 @@ def _setup_repo_registration_scheduled_job(self, pulp_server: PulpServer): pulp_server.name, pulp_server.repo_config_registration_regex_include, pulp_server.repo_config_registration_regex_exclude, + local_config_dir, ], result_ttl=172800, timeout=pulp_server.repo_config_registration_max_runtime, @@ -193,6 +197,7 @@ def _setup_repo_registration_scheduled_job(self, pulp_server: PulpServer): "pulp_server": pulp_server.name, "regex_include": pulp_server.repo_config_registration_regex_include, "regex_exclude": pulp_server.repo_config_registration_regex_exclude, + "local_repo_config_dir": local_config_dir, }, ) diff --git a/pulp_manager/app/services/pulp_manager.py b/pulp_manager/app/services/pulp_manager.py index 5d22c97..255702f 100644 --- a/pulp_manager/app/services/pulp_manager.py +++ b/pulp_manager/app/services/pulp_manager.py @@ -169,6 +169,11 @@ def _generate_base_path(self, name: str, base_url: str): if not base_url.endswith("/"): base_url = f"{base_url}/" + # If base_url already ends with the name (e.g., "int-demo-packages/" and name is "int-demo-packages"), + # don't duplicate it + if base_url.rstrip("/") == name: + return name + return self._process_package_name(name, base_url) def _process_package_name(self, name: str, base_url: str): @@ -667,6 +672,8 @@ def update_distribution( if repo_href and pulp_distribution.repository != repo_href: updates_needed = True pulp_distribution.repository = repo_href + # Clear publication when setting repository (can only have one or the other) + pulp_distribution.publication = None if updates_needed: log.debug( diff --git a/pulp_manager/app/services/reconciler.py b/pulp_manager/app/services/reconciler.py index 4600ec1..48621a4 100644 --- a/pulp_manager/app/services/reconciler.py +++ b/pulp_manager/app/services/reconciler.py @@ -64,25 +64,37 @@ def _get_pulp_server_repo_instances(self): distributions = get_all_distributions(client) repo_dict = {} - remote_dict = {} + remote_dict_by_href = {} + remote_dict_by_name = {} distribution_dict = {} for repo in repos: repo_dict[repo.name] = repo for remote in remotes: - remote_dict[remote.name] = remote + remote_dict_by_name[remote.name] = remote + remote_dict_by_href[remote.pulp_href] = remote for distribution in distributions: distribution_dict[distribution.name] = distribution repo_instances = {} for name, repo in repo_dict.items(): + # Get remote info from repo.remote if available, otherwise fall back to name matching + remote_href = None + remote_feed = None + if repo.remote and repo.remote in remote_dict_by_href: + remote_href = remote_dict_by_href[repo.remote].pulp_href + remote_feed = remote_dict_by_href[repo.remote].url + elif name in remote_dict_by_name: + remote_href = remote_dict_by_name[name].pulp_href + remote_feed = remote_dict_by_name[name].url + pulp_repo_instance = PulpRepoInstance( name, repo.pulp_href, - remote_dict[name].pulp_href if name in remote_dict else None, - remote_dict[name].url if name in remote_dict else None, + remote_href, + remote_feed, distribution_dict[name].pulp_href if name in distribution_dict else None ) repo_instances[name] = pulp_repo_instance diff --git a/pulp_manager/app/services/repo_config_register.py b/pulp_manager/app/services/repo_config_register.py index 685c689..dd89de3 100644 --- a/pulp_manager/app/services/repo_config_register.py +++ b/pulp_manager/app/services/repo_config_register.py @@ -7,6 +7,7 @@ import socket import tempfile import traceback +from contextlib import contextmanager from datetime import datetime from git import Repo @@ -43,18 +44,31 @@ def __init__(self, db: Session, name: str): job = get_current_job() self._job_id = job.id if job else None - def _clone_pulp_repo_config(self): - """Creates a temporary directory to clone the repo config defined in CONFIG. - Returns the path to the directory that was created + @contextmanager + def _get_repo_config_directory(self, local_path=None): + """Context manager that yields a config directory path. - :return: str - """ + If local_path is provided, yields it directly (no cleanup needed). + Otherwise, clones from git to a temp directory and cleans up on exit. - temp_dir = tempfile.mkdtemp(prefix="pulp_manager", dir="/tmp") - log.info(f"created {temp_dir} to clone repo config into") - Repo.clone_from(CONFIG["pulp"]["git_repo_config"], temp_dir) - log.info(f"clone into {temp_dir} completed") - return os.path.join(temp_dir, CONFIG["pulp"]["git_repo_config_dir"]) + :param local_path: Optional local filesystem path to config directory + :type local_path: str or None + :yield: str - Path to the config directory + """ + if local_path: + log.info(f"Using local repo config directory: {local_path}") + yield local_path + else: + temp_dir = tempfile.mkdtemp(prefix="pulp_manager", dir="/tmp") + try: + log.info(f"Created {temp_dir} to clone repo config into") + Repo.clone_from(CONFIG["pulp"]["git_repo_config"], temp_dir) + log.info(f"Clone into {temp_dir} completed") + config_dir = os.path.join(temp_dir, CONFIG["pulp"]["git_repo_config_dir"]) + yield config_dir + finally: + log.debug(f"Cleaning up cloned repo at {temp_dir}") + shutil.rmtree(temp_dir) #pylint:disable=line-too-long,too-many-branches def _generate_repo_config_from_file(self, file_path: str): @@ -217,14 +231,14 @@ def _parse_repo_config_files(self, repo_config_dir: str, regex_include: str, return parsed_repo_configs - def create_repos_from_git_config(self, regex_include: str=None, regex_exclude: str=None): - """Creates/updates repos on the target pulp server with repo config that is defined in git. - The repo to clone from is defined in the confi.ini + def create_repos_from_config(self, regex_include: str=None, regex_exclude: str=None, local_repo_config_dir: str=None): + """ + Creates/updates repos on the target pulp server with repo config that is defined in git or a local directory. + If local_repo_config_dir is provided, use it directly. Otherwise, clone from git. """ - current_repo = None task = self._task_crud.add(**{ - "name": f"{self._pulp_server_name} repo registartion", + "name": f"{self._pulp_server_name} repo registration", "date_started": datetime.utcnow(), "task_type": "repo_creation_from_git", "state": "running", @@ -232,24 +246,22 @@ def create_repos_from_git_config(self, regex_include: str=None, regex_exclude: s "worker_job_id": self._job_id, "task_args": { "regex_include": regex_include, - "regex_exclude": regex_exclude + "regex_exclude": regex_exclude, + "local_repo_config_dir": local_repo_config_dir } }) self._db.commit() - repo_config_dir = None - try: - repo_config_dir = self._clone_pulp_repo_config() - log.debug("repo cloned to {repo_config_dir}") - repo_configs = self._parse_repo_config_files( - repo_config_dir, regex_include, regex_exclude - ) + with self._get_repo_config_directory(local_repo_config_dir) as repo_config_dir: + repo_configs = self._parse_repo_config_files( + repo_config_dir, regex_include, regex_exclude + ) - for config in repo_configs: - log.debug(f"create/update repo for {config['name']}") - current_repo = config - self._pulp_manager.create_or_update_repository(**config) + for config in repo_configs: + log.debug(f"create/update repo for {config['name']}") + current_repo = config + self._pulp_manager.create_or_update_repository(**config) self._task_crud.update(task, **{ "state": "completed", "date_finished": datetime.utcnow() @@ -258,7 +270,7 @@ def create_repos_from_git_config(self, regex_include: str=None, regex_exclude: s except Exception: message = f"unexpected error occured registering repos on {self._pulp_server_name}" if current_repo: - message = f"failed to create/update repo for {config['name']}" + message = f"failed to create/update repo for {current_repo['name']}" log.error(message) log.error(traceback.format_exc()) @@ -274,7 +286,3 @@ def create_repos_from_git_config(self, regex_include: str=None, regex_exclude: s self._db.commit() raise - finally: - if repo_config_dir: - log.debug(f"tidying up cloned repo {repo_config_dir}") - shutil.rmtree(repo_config_dir) diff --git a/pulp_manager/app/tasks/repo_registration_task.py b/pulp_manager/app/tasks/repo_registration_task.py index a5127c2..eeebcc4 100644 --- a/pulp_manager/app/tasks/repo_registration_task.py +++ b/pulp_manager/app/tasks/repo_registration_task.py @@ -7,7 +7,8 @@ from pulp_manager.app.utils import log -def register_repos(pulp_server: str, regex_include: str=None, regex_exclude: str=None): +def register_repos(pulp_server: str, regex_include: str=None, regex_exclude: str=None, + local_repo_config_dir: str=None): """Task that is used to register repos on a pulp server :param pulp_server: name of the pulp server to register the repos for @@ -18,12 +19,17 @@ def register_repos(pulp_server: str, regex_include: str=None, regex_exclude: str with regex_exclude and regex_include. regex_exclude takes precendence and the repo will not be added to the pulp server :type regex_exclude: str + :param local_repo_config_dir: Optional local filesystem path to config directory. + If not provided, config will be cloned from git. + :type local_repo_config_dir: str """ db = session() try: repo_config_register = RepoConfigRegister(db, pulp_server) - repo_config_register.create_repos_from_git_config(regex_include, regex_exclude) + repo_config_register.create_repos_from_config( + regex_include, regex_exclude, local_repo_config_dir + ) except Exception: log.error(f"unexpected error registering repos for {pulp_server}") log.error(traceback.format_exc()) diff --git a/pulp_manager/tests/unit/services/test_pulp_manager.py b/pulp_manager/tests/unit/services/test_pulp_manager.py index 97f389c..5bafe8f 100644 --- a/pulp_manager/tests/unit/services/test_pulp_manager.py +++ b/pulp_manager/tests/unit/services/test_pulp_manager.py @@ -989,3 +989,24 @@ def test_process_package_name_complex_pattern(self): result = self.pulp_manager._process_package_name("acme-prod-webserver", "http://example.com/") assert result == "http://example.com/prod/acme/webserver" + + def test_generate_base_path_no_duplication(self): + """Tests that _generate_base_path doesn't duplicate when base_url equals name + """ + # When base_url equals name, should not duplicate + result = self.pulp_manager._generate_base_path("int-demo-packages", "int-demo-packages") + assert result == "int-demo-packages" + + # When base_url equals name with trailing slash, should not duplicate + result = self.pulp_manager._generate_base_path("int-demo-packages", "int-demo-packages/") + assert result == "int-demo-packages" + + def test_generate_base_path_with_different_base_url(self): + """Tests that _generate_base_path works correctly when base_url differs from name + """ + CONFIG["pulp"]["package_name_replacement_pattern"] = "" + CONFIG["pulp"]["package_name_replacement_rule"] = "" + + # When base_url is different from name, should concatenate + result = self.pulp_manager._generate_base_path("ext-centos7", "el7-x86_64") + assert result == "el7-x86_64/ext-centos7" diff --git a/pulp_manager/tests/unit/services/test_pulp_reconciler.py b/pulp_manager/tests/unit/services/test_pulp_reconciler.py index 0f93aa2..a7a09df 100644 --- a/pulp_manager/tests/unit/services/test_pulp_reconciler.py +++ b/pulp_manager/tests/unit/services/test_pulp_reconciler.py @@ -176,6 +176,53 @@ def new_pulp_client(pulp_server: PulpServer): assert deb_repo1.remote_feed is None assert deb_repo1.distribution_href == "/pulp/api/v3/distributions/deb/apt/3" + @patch("pulp_manager.app.services.reconciler.new_pulp_client") + @patch("pulp_manager.app.services.reconciler.get_all_repos") + @patch("pulp_manager.app.services.reconciler.get_all_remotes") + @patch("pulp_manager.app.services.reconciler.get_all_distributions") + def test_get_pulp_server_repo_instances_with_repo_remote_href( + self, mock_get_all_distributions, mock_get_all_remotes, + mock_get_all_repos, mock_new_pulp_client): + """Tests that remotes are correctly linked when repo.remote href is set, + even when the remote name differs from the repo name + """ + + def new_pulp_client(pulp_server: PulpServer): + return Pulp3Client(pulp_server.name, username=pulp_server.username, password="test") + + mock_new_pulp_client.side_effect = new_pulp_client + + # Repo has remote attribute set to a remote with different name + mock_get_all_repos.return_value = [ + Repository(**{ + "pulp_href": "/pulp/api/v3/repositories/rpm/rpm/1", + "name": "my-repo", + "remote": "/pulp/api/v3/remotes/rpm/rpm/different-remote" + }) + ] + + mock_get_all_remotes.return_value = [ + Remote(**{ + "pulp_href": "/pulp/api/v3/remotes/rpm/rpm/different-remote", + "name": "some-other-name", + "url": "https://href-matched-feed.domain.com", + "policy": "immediate" + }) + ] + + mock_get_all_distributions.return_value = [] + + result = self.pulp_reconciler._get_pulp_server_repo_instances() + + assert len(result) == 1 + + # Test href-based remote lookup (repo.remote set to different-named remote) + my_repo = result["my-repo"] + assert my_repo.name == "my-repo" + assert my_repo.repo_href == "/pulp/api/v3/repositories/rpm/rpm/1" + assert my_repo.remote_href == "/pulp/api/v3/remotes/rpm/rpm/different-remote" + assert my_repo.remote_feed == "https://href-matched-feed.domain.com" + def test_add_missing_repos(self): """Test that a list of repo instances get added to the db and a dict is returned containg the newly added entries diff --git a/pulp_manager/tests/unit/services/test_repo_config_register.py b/pulp_manager/tests/unit/services/test_repo_config_register.py index 8672f49..0a7b3ec 100644 --- a/pulp_manager/tests/unit/services/test_repo_config_register.py +++ b/pulp_manager/tests/unit/services/test_repo_config_register.py @@ -34,23 +34,36 @@ def teardown_method(self): engine.dispose() @patch("pulp_manager.app.services.repo_config_register.Repo.clone_from") - def test_clone_pulp_repo_config(self, mock_clone_from): - """Tests that a directory gets created which would contain checked out code from git + def test_get_repo_config_directory_from_git(self, mock_clone_from): + """Tests that the context manager clones from git and cleans up afterwards """ def clone_from(url, to_path): """Creates the repo_config directory in the to_path, as this would exist once the repo has been checked out """ - repo_config_path = os.path.join(to_path, "repo_config") os.mkdir(repo_config_path) mock_clone_from.side_effect = clone_from + temp_dir_created = None + + with self.repo_config_register._get_repo_config_directory() as config_dir: + assert os.path.isdir(config_dir) + # Store parent temp dir to verify cleanup + temp_dir_created = os.path.dirname(config_dir) + assert os.path.isdir(temp_dir_created) - git_clone_dir = self.repo_config_register._clone_pulp_repo_config() - assert os.path.isdir(git_clone_dir) - shutil.rmtree(git_clone_dir) + # Verify cleanup happened after context manager exits + assert not os.path.exists(temp_dir_created) + + def test_get_repo_config_directory_with_local_path(self): + """Tests that the context manager yields local path directly without cloning + """ + local_path = "/some/local/path" + + with self.repo_config_register._get_repo_config_directory(local_path) as config_dir: + assert config_dir == local_path @patch("pulp_manager.app.services.repo_config_register.os.path.isfile") @patch("pulp_manager.app.services.repo_config_register.HashiVaultClient.read_kv_secret") @@ -266,12 +279,96 @@ def test_apply_repo_name_prefix_neither(self): assert result == "myrepo" @patch("pulp_manager.app.services.repo_config_register.Repo.clone_from") - def test_create_repos_from_git_config_fail(self, mock_clone_from): + def test_create_repos_from_config_fail(self, mock_clone_from): """Tests logic flow that if they are errors an exception is raised """ mock_clone_from.side_effect = Exception("an error") - with pytest.raises(Exception): - self.repo_config_register.create_repos_from_git_config() + self.repo_config_register.create_repos_from_config() + + @patch("pulp_manager.app.services.repo_config_register.os.path.isfile") + @patch("pulp_manager.app.services.repo_config_register.os.walk") + def test_create_repos_from_config_with_local_dir(self, mock_os_walk, mock_isfile): + """Tests that create_repos_from_config uses local directory when provided + and does not attempt to clone from git + """ + + mock_isfile.return_value = True + mock_os_walk.return_value = [ + ('/local/config/remote', (), ('test-repo.json',)), + ] + + mock_open_data = { + "/local/config/remote/test-repo.json": json.dumps({ + "name": "test-repo", + "url": "https://example.com/repo", + "owner": "Test Owner", + "description": "Test repo", + "repo_type": "external", + "content_repo_type": "rpm", + "base_url": "test-x86_64" + }), + "/local/config/remote/global.json": json.dumps({ + "proxy": "http://proxy.example.com:8080" + }) + } + + def open_side_effect(name, mode=None): + return mock_open(read_data=mock_open_data.get(name, 'Default data'))() + + with patch("builtins.open", side_effect=open_side_effect): + # Should NOT clone from git when local_repo_config_dir is provided + with patch("pulp_manager.app.services.repo_config_register.Repo.clone_from") as mock_clone: + self.repo_config_register.create_repos_from_config( + local_repo_config_dir="/local/config" + ) + # Verify git clone was NOT called + mock_clone.assert_not_called() + # Verify repo was created via the mocked PulpManager + assert self.repo_config_register._pulp_manager.create_or_update_repository.called + + @patch("pulp_manager.app.services.repo_config_register.os.path.isfile") + @patch("pulp_manager.app.services.repo_config_register.os.walk") + @patch("pulp_manager.app.services.repo_config_register.Repo.clone_from") + def test_create_repos_from_config_with_git(self, mock_clone_from, mock_os_walk, mock_isfile): + """Tests that create_repos_from_config clones from git when local_repo_config_dir is not provided + """ + + def clone_from(url, to_path): + """Creates the repo_config directory in the to_path""" + repo_config_path = os.path.join(to_path, "repo_config") + os.mkdir(repo_config_path) + + mock_clone_from.side_effect = clone_from + mock_isfile.return_value = True + mock_os_walk.return_value = [ + ('/tmp/pulp_manager123/repo_config/remote', (), ('test-repo.json',)), + ] + + mock_open_data = { + "/tmp/pulp_manager123/repo_config/remote/test-repo.json": json.dumps({ + "name": "test-repo", + "url": "https://example.com/repo", + "owner": "Test Owner", + "description": "Test repo", + "repo_type": "external", + "content_repo_type": "rpm", + "base_url": "test-x86_64" + }), + "/tmp/pulp_manager123/repo_config/remote/global.json": json.dumps({ + "proxy": "http://proxy.example.com:8080" + }) + } + + def open_side_effect(name, mode=None): + return mock_open(read_data=mock_open_data.get(name, 'Default data'))() + + with patch("builtins.open", side_effect=open_side_effect): + # Should clone from git when local_repo_config_dir is NOT provided + self.repo_config_register.create_repos_from_config() + # Verify git clone WAS called + mock_clone_from.assert_called_once() + # Verify repo was created via the mocked PulpManager + assert self.repo_config_register._pulp_manager.create_or_update_repository.called diff --git a/pulp_manager/tests/unit/test_job_manager.py b/pulp_manager/tests/unit/test_job_manager.py index 78900c4..59b84ec 100644 --- a/pulp_manager/tests/unit/test_job_manager.py +++ b/pulp_manager/tests/unit/test_job_manager.py @@ -1,3 +1,4 @@ + """Tests for ensuring jobs are correct added to redis """ @@ -122,7 +123,7 @@ def test_setup_repo_registration_scheduled_job(self): assert len(jobs) == 1 job = jobs[0] - assert job.args == ["test_pulp_server_repo_registration", None, None] + assert job.args == ["test_pulp_server_repo_registration", None, None, None] assert job.meta["job_type"] == "REPO_REGISTRATION_SCHEDULED" assert job.meta["pulp_server"] == "test_pulp_server_repo_registration" assert job.meta["regex_include"] == None