diff --git a/.ci/nginx.conf.j2 b/.ci/nginx.conf.j2 index 70a2e5a3d..cf3695e8e 100644 --- a/.ci/nginx.conf.j2 +++ b/.ci/nginx.conf.j2 @@ -1,9 +1,6 @@ # Copy from pulp-oci-images. # Ideally we can get it upstream again. # -# TODO: Support IPv6. -# TODO: Maybe serve multiple `location`s, not just one. - # The "nginx" package on fedora creates this user and group. user nginx nginx; # Gunicorn docs suggest this value. @@ -24,10 +21,12 @@ http { # to build optimal hash types. types_hash_max_size 4096; + {%- if https | default(false) %} map $ssl_client_s_dn $ssl_client_s_dn_cn { default ""; ~CN=(?[^,]+) $CN; } + {%- endif %} upstream pulp-content { server 127.0.0.1:24816; @@ -85,7 +84,9 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; + {%- if https | default(false) %} proxy_set_header Remoteuser $ssl_client_s_dn_cn; + {%- endif %} # we don't want nginx trying to do something clever with # redirects, we set the Host: header above already. proxy_redirect off; @@ -136,6 +137,25 @@ http { try_files $uri $uri/ =404; } {%- endif %} + {% if https | default(false) -%} + location /oauth2token/ { + auth_basic "Tokens, Tokens, Tokens"; + auth_basic_user_file /etc/pulp/certs/oauth2passwd; + if ($request_method !~ POST) { + # This still triggers earlier than the auth_basic in the outer block. + return 403; + } + try_files /dev/null @oauth2token; + } + # Nginx "return" kicks in before basic_auth, so we must use it in a separate block. + # https://stackoverflow.com/questions/67975464/why-doesnt-basic-auth-work-with-a-simple-nginx-return-statement + location @oauth2token { + default_type application/json; + charset utf-8; + + return 200 '{"access_token": "DEADBEEF", "token_type": "bearer", "expires_in": 30}'; + } + {%- endif %} } {%- if https | default(false) %} server { diff --git a/.ci/run_container.sh b/.ci/run_container.sh index 0d7440fe1..b3a1c7e9b 100755 --- a/.ci/run_container.sh +++ b/.ci/run_container.sh @@ -50,7 +50,8 @@ else fi; mkdir -p "${PULP_CLI_TEST_TMPDIR}/settings/certs" -cp "${BASEPATH}/settings/settings.py" "${PULP_CLI_TEST_TMPDIR}/settings" +cp "${BASEPATH}/settings/settings.py" "${PULP_CLI_TEST_TMPDIR}/settings/settings.py" +echo "service_acct:$(openssl passwd secret)" > "${PULP_CLI_TEST_TMPDIR}/settings/certs/oauth2passwd" if [ -z "${PULP_HTTPS:+x}" ] then @@ -65,20 +66,18 @@ else export PULP_CA_BUNDLE="${PULP_CLI_TEST_TMPDIR}/settings/certs/ca.pem" ln -fs server.pem "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.crt" ln -fs server.key "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.key" - { - echo "AUTHENTICATION_BACKENDS = '@merge django.contrib.auth.backends.RemoteUserBackend'" - echo "MIDDLEWARE = '@merge django.contrib.auth.middleware.RemoteUserMiddleware'" - echo "REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = '@merge pulpcore.app.authentication.PulpRemoteUserAuthentication'" - echo "REMOTE_USER_ENVIRON_NAME = 'HTTP_REMOTEUSER'" - } >> "${PULP_CLI_TEST_TMPDIR}/settings/settings.py" fi export PULP_CONTENT_ORIGIN "${CONTAINER_RUNTIME}" \ run ${RM:+--rm} \ --env S6_KEEP_ENV=1 \ + ${OAS_VERSION:+--env PULP_SPECTACULAR_SETTINGS__OAS_VERSION="${OAS_VERSION}"} \ ${PULP_HTTPS:+--env PULP_HTTPS} \ + ${PULP_OAUTH2:+--env PULP_OAUTH2} \ ${PULP_API_ROOT:+--env PULP_API_ROOT} \ + ${PULP_DOMAIN_ENABLED:+--env PULP_DOMAIN_ENABLED} \ + ${PULP_ENABLED_PLUGINS:+--env PULP_ENABLED_PLUGINS} \ --env PULP_CONTENT_ORIGIN \ --detach \ --name "pulp-ephemeral" \ diff --git a/.ci/scripts/check_click_for_mypy.py b/.ci/scripts/check_click_for_mypy.py index e3b50f484..a905be7ef 100755 --- a/.ci/scripts/check_click_for_mypy.py +++ b/.ci/scripts/check_click_for_mypy.py @@ -1,11 +1,9 @@ #!/bin/env python3 -import sys - import click from packaging.version import parse if parse(click.__version__) < parse("8.1.1") or parse(click.__version__) >= parse("8.2"): print("🚧 Linting with mypy is currently only supported with click~=8.1.1. 🚧") print("🔧 Please run `pip install click~=8.1.1` first. 🔨") - sys.exit(1) + exit(1) diff --git a/.ci/settings/settings.py b/.ci/settings/settings.py index 6658028ac..ed037be22 100644 --- a/.ci/settings/settings.py +++ b/.ci/settings/settings.py @@ -1,3 +1,58 @@ +import os + ALLOWED_EXPORT_PATHS = ["/tmp"] ANALYTICS = False ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha256", "sha512"] + +if os.environ.get("PULP_HTTPS", "false").lower() == "true": + AUTHENTICATION_BACKENDS = "@merge django.contrib.auth.backends.RemoteUserBackend" + MIDDLEWARE = "@merge django.contrib.auth.middleware.RemoteUserMiddleware" + REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = ( + "@merge pulpcore.app.authentication.PulpRemoteUserAuthentication" + ) + REMOTE_USER_ENVIRON_NAME = "HTTP_REMOTEUSER" + +if os.environ.get("PULP_OAUTH2", "false").lower() == "true": + assert os.environ.get("PULP_HTTPS", "false").lower() == "true" + + def PulpCliFakeOauth2Authentication(*args, **kwargs): + # We need to lazy load this. + # Otherwise views may be instanciated, before this configuration is merged. + + from django.contrib.auth import authenticate + from drf_spectacular.extensions import OpenApiAuthenticationExtension + from rest_framework.authentication import BaseAuthentication + + class _PulpCliFakeOauth2Authentication(BaseAuthentication): + def authenticate(self, request): + auth_header = request.META.get("HTTP_AUTHORIZATION") + if auth_header == "Bearer DEADBEEF": + return authenticate(request, remote_user="admin"), None + else: + return None + + def authenticate_header(self, request): + return 'Bearer realm="Pulp"' + + class PulpCliFakeOauth2AuthenticationScheme(OpenApiAuthenticationExtension): + target_class = _PulpCliFakeOauth2Authentication + name = "PulpCliFakeOauth2" + + def get_security_definition(self, auto_schema): + return { + "type": "oauth2", + "flows": { + "clientCredentials": { + "tokenUrl": "https://localhost:8080/oauth2token/", + "scopes": {"api.console": "grant_access_to_pulp"}, + }, + }, + } + + return _PulpCliFakeOauth2Authentication(*args, **kwargs) + + PULP_CLI_FAKE_OAUTH2_AUTHENTICATION = PulpCliFakeOauth2Authentication + + REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = ( + "@merge pulpcore.app.settings.PULP_CLI_FAKE_OAUTH2_AUTHENTICATION" + ) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 29d3be557..92f3ccfed 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,23 +1,25 @@ +--- version: 2 updates: -- package-ecosystem: pip +- package-ecosystem: "pip" directory: "/" schedule: - interval: daily + interval: "daily" commit-message: prefix: "[PIP] " open-pull-requests-limit: 10 -- package-ecosystem: pip +- package-ecosystem: "pip" directory: "/pulp-glue" schedule: - interval: daily + interval: "daily" commit-message: prefix: "[PIP] " open-pull-requests-limit: 10 -- package-ecosystem: github-actions +- package-ecosystem: "github-actions" directory: "/" schedule: - interval: weekly + interval: "weekly" commit-message: prefix: "[GHA] " open-pull-requests-limit: 10 +... diff --git a/.github/workflows/cookiecutter.yml b/.github/workflows/cookiecutter.yml new file mode 100644 index 000000000..96546b8ee --- /dev/null +++ b/.github/workflows/cookiecutter.yml @@ -0,0 +1,58 @@ +--- +name: "Update CI from cookiecutter" +on: + workflow_dispatch: + +defaults: + run: + working-directory: "pulp-cli" + +jobs: + update-ci: + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@v4" + with: + repository: "pulp/pulp-cli" + path: "pulp-cli" + - uses: "actions/checkout@v4" + with: + token: "${{ secrets.RELEASE_TOKEN }}" + path: "pulp-cli" + - name: "Set up Python" + uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Setup git" + run: | + git config user.name pulpbot + git config user.email pulp-infra@redhat.com + - name: "Install python dependencies" + run: | + pip install cookiecutter tomlkit + - name: "Apply cookiecutter templates" + run: | + ../pulp-cli/cookiecutter/apply_templates.py + if [ "$(git status --porcelain)" ] + then + git add . + git commit -m "Update cookiecutter" + fi + - name: "Create Pull Request" + uses: "peter-evans/create-pull-request@v7" + id: "create_pr" + with: + token: "${{ secrets.RELEASE_TOKEN }}" + title: "Update cookiecutter" + body: "" + branch: "update_cookiecutter" + delete-branch: true + path: "pulp-cli" + - name: "Mark PR automerge" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr.outputs.pull-request-number }}" + if: "steps.create_pr.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true +... diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 96e6dcbe0..62b1c0577 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,7 +6,7 @@ on: jobs: test: if: "endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-24.04" steps: - uses: "actions/checkout@v4" - uses: "actions/cache@v4" @@ -19,7 +19,7 @@ jobs: - name: "Set up Python" uses: "actions/setup-python@v5" with: - python-version: "3.11" + python-version: "3.13" - name: "Install Test Dependencies" run: | pip install -r doc_requirements.txt @@ -28,7 +28,7 @@ jobs: make docs no-test: if: "!endsWith(github.base_ref, 'main')" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-latest" steps: - run: | echo "Skip docs testing on non-main branches." diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8a6005a51..ec4f4bfb0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: matrix: python: - "3.11" - - "3.12" + - "3.13" steps: - uses: "actions/checkout@v4" - uses: "actions/cache@v4" diff --git a/.github/workflows/release_branch.yml b/.github/workflows/release_branch.yml index ea1e9192a..9d23a4730 100644 --- a/.github/workflows/release_branch.yml +++ b/.github/workflows/release_branch.yml @@ -8,6 +8,8 @@ jobs: runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" + with: + token: "${{ secrets.RELEASE_TOKEN }}" - name: "Set up Python" uses: "actions/setup-python@v5" with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d7e2355c..64704c0bf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ env: jobs: test: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-24.04" strategy: fail-fast: false matrix: @@ -60,6 +60,10 @@ jobs: if [ "${{matrix.lower_bounds}}" ] then pip install dist/pulp_cli-*.whl pulp-glue/dist/pulp_glue-*.whl -r test_requirements.txt -c lower_bounds_constraints.lock + elif [ "${{matrix.upper_bounds}}" ] + then + .ci/scripts/calc_constraints.py pyproject.toml pulp-glue/pyproject.toml --upper > upper_bounds_constraints.lock + pip install dist/pulp_cli-*.whl pulp-glue/dist/pulp_glue-*.whl -r test_requirements.txt -c upper_bounds_constraints.lock else pip install dist/pulp_cli-*.whl pulp-glue/dist/pulp_glue-*.whl -r test_requirements.txt fi @@ -70,7 +74,11 @@ jobs: FROM_TAG: "${{ matrix.from_tag }}" CONTAINER_FILE: "${{ matrix.container_file }}" PULP_HTTPS: "${{ matrix.pulp_https }}" + PULP_OAUTH2: "${{ matrix.pulp_oauth2 }}" PULP_API_ROOT: "${{ matrix.pulp_api_root }}" + PULP_DOMAIN_ENABLED: "${{ matrix.pulp_domain_enabled }}" + PULP_ENABLED_PLUGINS: "${{ matrix.pulp_enabled_plugins }}" + OAS_VERSION: "${{ matrix.oas_version }}" run: | .ci/run_container.sh make test ... diff --git a/CHANGES/+click.dependency.bugfix b/CHANGES/+click.dependency.bugfix new file mode 100644 index 000000000..7089b167f --- /dev/null +++ b/CHANGES/+click.dependency.bugfix @@ -0,0 +1 @@ +Adjust click dependency constraints to breaking changes in y-releases. diff --git a/Makefile b/Makefile index 482777daf..7f65567f1 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,9 @@ build: cd pulp-glue; pyproject-build -n pyproject-build -n -black: +black: format + +format: isort . cd pulp-glue; isort . black . @@ -36,11 +38,20 @@ tests/cli.toml: test: | tests/cli.toml python3 -m pytest -v tests pulp-glue/tests +livetest: | tests/cli.toml + python3 -m pytest -v tests pulp-glue/tests -m live + +unittest: + python3 -m pytest -v tests pulp-glue/tests -m "not live" + +unittest_glue: + python3 -m pytest -v pulp-glue/tests -m "not live" + docs: pulp-docs build servedocs: - pulp-docs serve -w CHANGES.md -w pulp-glue/pulp_glue -w pulpcore/cli/common/generic.py + pulp-docs serve -w CHANGES.md -w pulp-glue/pulp_glue -w pulp_cli/generic.py pulp-glue/pulp_glue/%/locale/messages.pot: pulp-glue/pulp_glue/%/*.py xgettext -d $* -o $@ pulp-glue/pulp_glue/$*/*.py diff --git a/pulp-glue/pyproject.toml b/pulp-glue/pyproject.toml index 3ee29db4b..5880ecaeb 100644 --- a/pulp-glue/pyproject.toml +++ b/pulp-glue/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools<81"] build-backend = "setuptools.build_meta" [project] @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "importlib_resources>=5.4.0,<6.2;python_version<'3.9'", - "packaging>=20.0,<=24.2", # CalVer + "packaging>=20.0,<=25.0", # CalVer "requests>=2.24.0,<2.33", ] diff --git a/pyproject.toml b/pyproject.toml index 920df48fe..b7a3d1290 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools<73"] +requires = ["setuptools<81"] build-backend = "setuptools.build_meta" [project] @@ -24,8 +24,8 @@ classifiers=[ ] dependencies = [ "pulp-glue==0.29.4.dev", - "click>=8.0.0,<9", - "packaging>=20.0,<=24.2", # CalVer + "click>=8.0.0,<8.2", # Proven to not do semver. + "packaging>=20.0,<=25", # CalVer "PyYAML>=5.3,<6.1", "schema>=0.7.5,<0.8", "toml>=0.10.2,<0.11", @@ -76,6 +76,7 @@ docs = true translations = true main_package = "common" binary_dependencies = "" +unittests = false [tool.bumpversion] # This section is managed by the cookiecutter templates. @@ -193,7 +194,7 @@ showcontent = true # This section is managed by the cookiecutter templates. directory = "misc" name = "Misc" -showcontent = false +showcontent = true [tool.black] @@ -205,7 +206,7 @@ exclude = "cookiecutter" # This section is managed by the cookiecutter templates. profile = "black" line_length = 100 -skip = ["pulp-glue", "cookiecutter"] +extend_skip = ["pulp-glue", "cookiecutter"] [tool.flake8] # This section is managed by the cookiecutter templates. diff --git a/test_requirements.txt b/test_requirements.txt index ffe4acc91..27e883bfd 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,6 +2,7 @@ pytest>=7.0.0,<8.4 pytest-subtests>=0.12.0,<0.14 python-gnupg>=0.5.0,<0.6 +jinja2>=3.1.4,<3.2 trustme>=1.1.0,<1.2 # No pinning here, because we only switch on optional dependencies here.