diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..c061c95fff --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,31 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: "monthly" + groups: + "github actions": + patterns: + - "*" + - package-ecosystem: pip + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 10 + groups: + pytest: + patterns: + - "pytest*" + - "hypothesis" + pyinstaller: + patterns: + - "pyinstaller*" + mypy: + patterns: + - "types-*" + - "mypy" + openssl: + patterns: + - "pyopenssl" + - "cryptography" diff --git a/.github/node-version.txt b/.github/node-version.txt index 8351c19397..3c032078a4 100644 --- a/.github/node-version.txt +++ b/.github/node-version.txt @@ -1 +1 @@ -14 +18 diff --git a/.github/python-version.txt b/.github/python-version.txt index 2c0733315e..e4fba21835 100644 --- a/.github/python-version.txt +++ b/.github/python-version.txt @@ -1 +1 @@ -3.11 +3.12 diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 0000000000..c2ae2daec4 --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,40 @@ +name: autofix.ci + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + autofix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version-file: .github/python-version.txt + - run: pip install -e .[dev] + - run: ruff check --fix-only . + - run: ruff format . + + - run: web/gen/all + + - uses: actions/setup-node@v4 + with: + node-version-file: .github/node-version.txt + - run: npm ci + working-directory: web + - run: npm run eslint + working-directory: web + continue-on-error: true + - run: npm run prettier + working-directory: web + + - uses: mhils/add-pr-ref-in-changelog@main + + - uses: autofix-ci/action@dd55f44df8f7cdb7a6bf74c78677eb8acd40cd0a diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93b4d32967..0b7a709d1f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,89 +1,68 @@ name: CI -on: [ push, pull_request, workflow_dispatch ] +on: + push: + branches: + - '**' + - '!dependabot/**' + - '!*-patch-*' + pull_request: + merge_group: + workflow_dispatch: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - lint-pr: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - uses: TrueBrain/actions-flake8@c2deca24d388aa5aedd6478332aa9df4600b5eac # v2.1 - # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions - lint-local: - if: github.event_name == 'push' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e flake8 + lint: + uses: mhils/workflows/.github/workflows/python-tox.yml@v11 + with: + cmd: tox -e lint || true + filename-matching: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox + uses: mhils/workflows/.github/workflows/python-tox.yml@v11 + with: + cmd: tox -e filename_matching || true + mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e mypy + uses: mhils/workflows/.github/workflows/python-tox.yml@v11 + with: + cmd: tox -e mypy || true + individual-coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e individual_coverage + uses: mhils/workflows/.github/workflows/python-tox.yml@v11 + with: + cmd: tox -e individual_coverage || true + test: strategy: fail-fast: false matrix: include: - os: ubuntu-latest - py: "3.11" + py: "3.13" - os: windows-latest - py: "3.11" + py: "3.13" - os: macos-latest + py: "3.13" + - os: ubuntu-latest + py: "3.12" + - os: ubuntu-latest py: "3.11" - os: ubuntu-latest py: "3.10" runs-on: ${{ matrix.os }} steps: - run: printenv - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} - run: pip install tox @@ -96,64 +75,122 @@ jobs: # run tests with loopback only. We need to sudo for unshare, which means we need an absolute path for tox. sudo unshare --net -- sh -c "ip link set lo up; $(which tox) -e py" if: matrix.os == 'ubuntu-latest' - - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 - # mirrored below and at https://github.com/mitmproxy/mitmproxy/settings/actions + - uses: codecov/codecov-action@v4 with: - file: ./coverage.xml - name: ${{ matrix.os }} + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + + test-old-dependencies: + if: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version-file: .github/python-version.txt + - run: pip install tox-uv + - run: tox -e old-dependencies build: strategy: fail-fast: false matrix: include: - - image: macos-11 - platform: macos + - image: macos-14 + platform: macos-arm64 + - image: macos-13 + platform: macos-x86_64 - image: windows-2019 platform: windows - image: ubuntu-20.04 # Oldest available version so we get oldest glibc possible. platform: linux runs-on: ${{ matrix.image }} - env: - CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: .github/python-version.txt - - if: matrix.platform == 'windows' - uses: actions/cache@v3 - with: - path: release/installbuilder/setup - key: installbuilder - - run: pip install -e .[dev] - - run: python -u release/build.py standalone-binaries + - run: pip install .[dev] # pyinstaller 5.9 does not like pyproject.toml + editable installs. + + - if: startsWith(matrix.platform, 'macos') && github.repository == 'mitmproxy/mitmproxy' + && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) + id: keychain + uses: apple-actions/import-codesign-certs@63fff01cd422d4b7b855d40ca1e9d34d2de9427d + with: + keychain: ${{ runner.temp }}/temp + p12-file-base64: ${{ secrets.APPLE_CERTIFICATE }} + p12-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + - if: startsWith(matrix.platform, 'macos') && github.repository == 'mitmproxy/mitmproxy' + && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) + run: | + python -u release/build.py macos-app \ + --keychain "${{ runner.temp }}/temp.keychain" \ + --team-id "S8XHQB96PW" \ + --apple-id "${{ secrets.APPLE_ID }}" \ + --password "${{ secrets.APPLE_APP_PASSWORD }}" + + # Linux - if: matrix.platform == 'linux' - run: python -u release/build.py --dirty wheel - - if: matrix.platform == 'windows' && github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' - run: python -u release/build.py --dirty installbuilder-installer msix-installer - - uses: actions/upload-artifact@v3 + run: python -u release/build.py standalone-binaries wheel + + # Windows + - if: matrix.platform == 'windows' + run: python -u release/build.py standalone-binaries + + - uses: actions/upload-artifact@v4 with: - # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24 name: binaries.${{ matrix.platform }} - path: | - release/dist + path: release/dist + + build-wheel: + uses: mhils/workflows/.github/workflows/python-build.yml@v11 + with: + python-version-file: .github/python-version.txt + artifact: binaries.wheel + + build-windows-installer: + runs-on: windows-latest + if: github.repository == 'mitmproxy/mitmproxy' && ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/citest' || + startsWith(github.ref, 'refs/tags/') + ) + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version-file: .github/python-version.txt + + - run: pip install .[dev] # pyinstaller 5.9 does not like pyproject.toml + editable installs. + - run: python -u release/build.py installbuilder-installer msix-installer + env: + CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }} + + - uses: actions/upload-artifact@v4 + with: + name: binaries.windows-installer + path: release/dist test-web-ui: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false - - run: git rev-parse --abbrev-ref HEAD - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version-file: .github/node-version.txt - name: Cache Node.js modules - uses: actions/cache@v3 + uses: actions/cache@v4 with: # npm cache files are stored in `~/.npm` on Linux/macOS path: ~/.npm @@ -165,19 +202,34 @@ jobs: run: npm ci - working-directory: ./web run: npm test - - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 - # mirrored above and at https://github.com/mitmproxy/mitmproxy/settings/actions + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./web/coverage/coverage-final.json + + test-docker: + runs-on: ubuntu-latest + needs: build-wheel + steps: + - uses: actions/checkout@v4 with: - file: ./web/coverage/coverage-final.json - name: web + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: binaries.wheel + path: release/docker + - name: Build container + run: docker build --tag localtesting release/docker + - name: Test container + run: docker run --rm -v $PWD/release:/release localtesting mitmdump -s /release/selftest.py docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: .github/python-version.txt - run: | @@ -186,7 +238,7 @@ jobs: sudo dpkg -i hugo*.deb - run: pip install -e .[dev] - run: ./docs/build.py - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: docs path: docs/public @@ -196,54 +248,110 @@ jobs: env: DOCS_ARCHIVE: true - if: startsWith(github.ref, 'refs/tags/') - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docs-archive path: docs/public + check: + if: always() + needs: +# - lint +# - filename-matching +# - mypy +# - individual-coverage + - test + - test-docker +# - test-old-dependencies + - test-web-ui + - build + - build-wheel + - build-windows-installer + - docs + uses: mhils/workflows/.github/workflows/alls-green.yml@v11 + with: + jobs: ${{ toJSON(needs) }} + allowed-skips: build-windows-installer + # Separate from everything else because slow. - build-and-deploy-docker: + deploy-docker: if: github.repository == 'mitmproxy/mitmproxy' && ( - github.ref == 'refs/heads/main' - || github.ref == 'refs/heads/citest' - || startsWith(github.ref, 'refs/tags/') + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/citest' || + startsWith(github.ref, 'refs/tags/') ) + permissions: + id-token: write + attestations: write + packages: write environment: deploy-docker - needs: - - test - - test-web-ui - - build - - docs + needs: check runs-on: ubuntu-latest - env: - DOCKER_USERNAME: mitmbot - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v4 + - uses: actions/download-artifact@v4 with: - python-version-file: .github/python-version.txt - - uses: actions/download-artifact@v2 + name: binaries.wheel + path: release/docker + - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v1.6.0 + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: - name: binaries.linux - path: release/dist - - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1.2.0 - - uses: docker/setup-buildx-action@b1f1f719c7cd5364be7c82e366366da322d01f7c # v1.6.0 - - run: python release/build-and-deploy-docker.py + username: mitmbot + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to GitHub Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + with: + images: | + mitmproxy/mitmproxy + ghcr.io/mitmproxy/mitmproxy + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=dev,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=citest,enable=${{ github.ref == 'refs/heads/citest' }} + + - name: Build and push + id: push + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + with: + context: release/docker + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + - uses: actions/attest-build-provenance@v1 + with: + subject-name: ghcr.io/${{ github.repository }} + subject-digest: ${{ steps.push.outputs.digest }} deploy: # This action has access to our AWS keys, so we are extra careful here. # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks. if: github.repository == 'mitmproxy/mitmproxy' && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) environment: ${{ (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/')) && 'deploy-release' || 'deploy-snapshot' }} - needs: - - test - - test-web-ui - - build - - docs + needs: check runs-on: ubuntu-latest + permissions: + id-token: write + attestations: write env: # PyPI and MSFT keys are only available for the deploy-release environment # The AWS access key for snapshots is scoped to branches/* as well. @@ -256,11 +364,14 @@ jobs: MSFT_TENANT_ID: ${{ secrets.MSFT_TENANT_ID }} MSFT_CLIENT_ID: ${{ secrets.MSFT_CLIENT_ID }} MSFT_CLIENT_SECRET: ${{ secrets.MSFT_CLIENT_SECRET }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: .github/python-version.txt - run: sudo apt-get update @@ -268,27 +379,27 @@ jobs: - if: startsWith(github.ref, 'refs/tags/') run: sudo apt-get install -y twine - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: docs path: docs/public - if: startsWith(github.ref, 'refs/tags/') - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: docs-archive path: docs/archive - - uses: actions/download-artifact@v3 - with: - name: binaries.windows - path: release/dist - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: binaries.linux + pattern: binaries.* + merge-multiple: true path: release/dist - - uses: actions/download-artifact@v3 + - id: provenance + uses: actions/attest-build-provenance@v1 with: - name: binaries.macos - path: release/dist + subject-path: 'release/dist/*' + - run: | + REF=${{ github.ref_name }} + mv ${{ steps.provenance.outputs.bundle-path }} release/dist/mitmproxy-${REF#v}.sigstore - run: ls docs/public - run: ls release/dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e32fd6d1a5..161d80e7b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,13 +22,13 @@ jobs: environment: deploy-release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.GH_PUSH_TOKEN }} # this token works to push to the protected main branch. - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version-file: .github/node-version.txt - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: .github/python-version.txt - run: ./release/release.py ${{ inputs.version }} ${{ inputs.skip-branch-status-check }} diff --git a/.gitignore b/.gitignore index 2a6f5eb373..a5dfafcc27 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ MANIFEST **/tmp /venv* +/.venv* *.py[cdo] *.swp *.swo diff --git a/CHANGELOG.md b/CHANGELOG.md index 5514361627..845a1c82ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,437 @@ # Release History + + ## Unreleased: mitmproxy next +- Stop sorting keys in JSON contentview + ([#7346](https://github.com/mitmproxy/mitmproxy/pull/7346), @injust) + +## 24 November 2024: mitmproxy 11.0.1 + +- Tighten HTTP detection heuristic to better support custom TCP-based protocols. + ([#7228](https://github.com/mitmproxy/mitmproxy/pull/7228), @fatanugraha) +- Implement stricter validation of HTTP headers to harden against request smuggling attacks. + ([#7345](https://github.com/mitmproxy/mitmproxy/issues/7345), @mhils) +- Increase HTTP/2 default flow control window size, fixing performance issues. + ([#7317](https://github.com/mitmproxy/mitmproxy/pull/7317), @sujaldev) +- Fix a bug where mitmproxy would incorrectly report that TLS 1.0 and 1.1 are not supported + with the current OpenSSL build. + ([#7241](https://github.com/mitmproxy/mitmproxy/pull/7241), @mhils) +- Docker: Update image to Python 3.13 on Debian Bookworm. + ([#7242](https://github.com/mitmproxy/mitmproxy/pull/7242), @mhils) +- Add a `tun` proxy mode that creates a virtual network device on Linux for transparent proxying. + ([#7278](https://github.com/mitmproxy/mitmproxy/pull/7278), @mhils) +- `browser.start` command now supports Firefox. + ([#7239](https://github.com/mitmproxy/mitmproxy/pull/7239), @sujaldev) +- Fix interaction of the `modify_headers` and `stream_large_bodies` options. + This may break users of `modify_headers` that rely on filters referencing the message body. + We expect this to be uncommon, but please make yourself heard if that's not the case. + ([#7286](https://github.com/mitmproxy/mitmproxy/pull/7286), @lukant) +- Fix a crash when handling corrupted compressed body in savehar addon and its tests. + ([#7320](https://github.com/mitmproxy/mitmproxy/pull/7320), @8192bytes) +- Remove dependency on `protobuf` library as it was no longer being used. + ([#7327](https://github.com/mitmproxy/mitmproxy/pull/7327), @matthew16550) + +## 02 October 2024: mitmproxy 11.0.0 + +- mitmproxy now supports transparent HTTP/3 proxying. + ([#7202](https://github.com/mitmproxy/mitmproxy/pull/7202), @errorxyz, @meitinger, @mhils) +- Add HTTP3 support in HTTPS reverse-proxy mode. + ([#7114](https://github.com/mitmproxy/mitmproxy/pull/7114), @errorxyz) +- mitmproxy now officially supports Python 3.13. + ([#6934](https://github.com/mitmproxy/mitmproxy/pull/6934), @mhils) +- Tighten HTTP detection heuristic to better support custom TCP-based protocols. + ([#7087](https://github.com/mitmproxy/mitmproxy/pull/7087)) +- Add `show_ignored_hosts` option to display ignored flows in the UI. + This option is implemented as a temporary workaround and will be removed in the future. + ([#6720](https://github.com/mitmproxy/mitmproxy/pull/6720), @NicolaiSoeborg) +- Fix slow tnetstring parsing in case of very large tnetstring. + ([#7121](https://github.com/mitmproxy/mitmproxy/pull/7121), @mik1904) +- Add `getaddrinfo`-based fallback for DNS resolution if we are unable to + determine the operating system's name servers. + ([#7122](https://github.com/mitmproxy/mitmproxy/pull/7122), @mhils) +- Improve the error message when users specify the `certs` option without a matching private key. + ([#7073](https://github.com/mitmproxy/mitmproxy/pull/7073), @mhils) +- Fix a bug where intermediate certificates would not be transmitted when using QUIC. + ([#7073](https://github.com/mitmproxy/mitmproxy/pull/7073), @mhils) +- Fix a bug where fragmented QUIC client hellos were not handled properly. + ([#7067](https://github.com/mitmproxy/mitmproxy/pull/7067), @errorxyz) +- Emit a warning when users configure a TLS version that is not supported by the + current OpenSSL build. + ([#7139](https://github.com/mitmproxy/mitmproxy/pull/7139), @mhils) +- Fix a bug where mitmproxy would crash when receiving `STOP_SENDING` QUIC frames. + ([#7119](https://github.com/mitmproxy/mitmproxy/pull/7119), @mhils) +- Fix error when unmarking all flows. + ([#7192](https://github.com/mitmproxy/mitmproxy/pull/7192), @bburky) +- Add addon to update the alt-svc header in reverse mode. + ([#7093](https://github.com/mitmproxy/mitmproxy/pull/7093), @errorxyz) +- Do not send unnecessary empty data frames when streaming HTTP/2. + ([#7196](https://github.com/mitmproxy/mitmproxy/pull/7196), @rubu) +- Fix a bug where mitmproxy would ignore Ctrl+C/SIGTERM on OpenBSD. + ([#7130](https://github.com/mitmproxy/mitmproxy/pull/7130), @catap) +- Fix of measurement unit in HAR import, duration is in milliseconds. + ([#7179](https://github.com/mitmproxy/mitmproxy/pull/7179), @dstd) +- `Connection.tls_version` now is `QUICv1` instead of `QUIC` for QUIC. + ([#7201](https://github.com/mitmproxy/mitmproxy/pull/7201), @mhils) +- Add support for full mTLS with client certs between client and mitmproxy. + ([#7175](https://github.com/mitmproxy/mitmproxy/pull/7175), @Kriechi) +- Update documentation adding a list of all possibile web_columns. + ([#7205](https://github.com/mitmproxy/mitmproxy/pull/7205), @lups2000, @Abhishek-Bohora) + +## 02 August 2024: mitmproxy 10.4.2 + +- Fix a crash on startup when mitmproxy is unable to determine the OS' DNS servers + ([#7066](https://github.com/mitmproxy/mitmproxy/pull/7066), @errorxyz) + +## 29 July 2024: mitmproxy 10.4.1 + +- Fix a bug where macOS local mode would not start up on macOS. + ([#7045](https://github.com/mitmproxy/mitmproxy/pull/7045), @mhils) +- Fix UDP error handling when we learn that the remote has disconnected. + ([#7045](https://github.com/mitmproxy/mitmproxy/pull/7045), @mhils) +- Container images are now published to both Docker Hub and GitHub Container Registry. + ([#7061](https://github.com/mitmproxy/mitmproxy/pull/7061), @mhils) + +## 25 July 2024: mitmproxy 10.4.0 + +* Add support for DNS over TCP. + ([#6935](https://github.com/mitmproxy/mitmproxy/pull/6935), @errorxyz) +* Add first MVP new Capture Tab in mitmweb + ([#6999](https://github.com/mitmproxy/mitmproxy/pull/6999), @lups2000) +* Add `HttpConnectedHook` and `HttpConnectErrorHook`. + ([#6930](https://github.com/mitmproxy/mitmproxy/pull/6930), @errorxyz) +* Fix non-linear growth in processing time for large HTTP bodies. + ([#6952](https://github.com/mitmproxy/mitmproxy/pull/6952), @jackfromeast) +* Fix a bug where connections would be incorrectly ignored with `allow_hosts`. + ([#7002](https://github.com/mitmproxy/mitmproxy/pull/7002), @JarLob, @mhils) +* Fix zstd decompression to read across frames. + ([#6921](https://github.com/mitmproxy/mitmproxy/pull/6921), @zendai) +* Handle certificates we cannot parse more gracefully. + ([#6994](https://github.com/mitmproxy/mitmproxy/pull/6994), @mhils) +* Parse compressed domain names in ResourceRecord data. + ([#6954](https://github.com/mitmproxy/mitmproxy/pull/6954), @errorxyz) +* Fix a bug where mitmweb's flow list would not stay at the bottom. + ([#7008](https://github.com/mitmproxy/mitmproxy/pull/7008), @mhils) +* Fix a bug where SSH connections would be incorrectly handled as HTTP. + ([#7041](https://github.com/mitmproxy/mitmproxy/pull/7041), @mhils) +* Skip UTF-8 byte-order marks (BOM) when loading HAR files. + ([#6897](https://github.com/mitmproxy/mitmproxy/pull/6897), @dstd) +* Allow `typing.Sequence[str]` to be an editable option. + ([#7001](https://github.com/mitmproxy/mitmproxy/pull/7001), @errorxyz) +* Add Host header to CONNECT requests. + ([#7021](https://github.com/mitmproxy/mitmproxy/pull/7021), @petsneakers) +* Support all query types in DNS mode. + ([#6975](https://github.com/mitmproxy/mitmproxy/pull/6975), @errorxyz) +* Fix a bug where mitmproxy would crash for pipelined HTTP flows. + ([#7031](https://github.com/mitmproxy/mitmproxy/pull/7031), @gdiepen, @mhils) +* Add an optional "index" column for mitmweb. + ([#7039](https://github.com/mitmproxy/mitmproxy/pull/7039), @mhils) + +## 12 June 2024: mitmproxy 10.3.1 + +* Release tags are now prefixed with `v` again. + ([#6810](https://github.com/mitmproxy/mitmproxy/pull/6810), @mhils) +* Fix a bug where mitmproxy would not exit when `-n` is passed. + ([#6819](https://github.com/mitmproxy/mitmproxy/pull/6819), @mhils) +* Set the `unbuffered` (stdout/stderr) flag for the `mitmdump` PyInstaller build. + ([#6821](https://github.com/mitmproxy/mitmproxy/pull/6821), @Prinzhorn) +* Fix a bug where client replay would not work with proxyauth. + ([#6866](https://github.com/mitmproxy/mitmproxy/pull/6866), @mhils) +* Fix slowdown when sending large amounts of data over HTTP/2. + ([#6875](https://github.com/mitmproxy/mitmproxy/pull/6875), @aib) +* Add an option to strip HTTPS records from DNS responses to block encrypted ClientHellos. + ([#6876](https://github.com/mitmproxy/mitmproxy/pull/6876), @errorxyz) +* Add an API to parse HTTPS records from DNS RDATA. + ([#6884](https://github.com/mitmproxy/mitmproxy/pull/6884), @errorxyz) +* Fix flow export in mitmweb for Safari + ([#6917](https://github.com/mitmproxy/mitmproxy/pull/6917), @mhils, @canyesilyurt) +* Releases now come with a Sigstore attestations file to demonstrate build provenance. + ([f05c050](https://github.com/mitmproxy/mitmproxy/commit/f05c050f615b9ab9963707944c893bc94e738525), @mhils) + +## 17 April 2024: mitmproxy 10.3.0 + +* Add support for editing non text files in a hex editor + ([#6768](https://github.com/mitmproxy/mitmproxy/pull/6768), @wnyyyy) +* Add `server_connect_error` hook that is triggered when connection establishment fails. + ([#6806](https://github.com/mitmproxy/mitmproxy/pull/6806), @haanhvu, @spacewasp, @mhils) +* Add section in mitmweb for rendering, adding and removing a comment + ([#6709](https://github.com/mitmproxy/mitmproxy/pull/6709), @lups2000) +* Fix multipart form content view being unusable. + ([#6653](https://github.com/mitmproxy/mitmproxy/pull/6653), @DaniElectra) +* Documentation Improvements on CA Certificate Generation + ([#5370](https://github.com/mitmproxy/mitmproxy/pull/5370), @zioalex) +* Make it possible to read flows from stdin with mitmweb. + ([#6732](https://github.com/mitmproxy/mitmproxy/pull/6732), @jaywor1) +* Update aioquic dependency to >= 1.0.0, < 2.0.0. + ([#6747](https://github.com/mitmproxy/mitmproxy/pull/6747), @jlaine) +* Fix a bug where async `client_connected` handlers would crash mitmproxy. + ([#6749](https://github.com/mitmproxy/mitmproxy/pull/6749), @mhils) +* Add button to close flow details panel + ([#6734](https://github.com/mitmproxy/mitmproxy/pull/6734), @lups2000) +* Ignore SIGPIPE signals when there is lots of traffic. + Socket errors are handled directly and do not require extra signals + that generate noise. + ([#6764](https://github.com/mitmproxy/mitmproxy/pull/6764), @changsin) +* Add primitive websocket interception and modification + ([#6766](https://github.com/mitmproxy/mitmproxy/pull/6766), @errorxyz) +* Add support for exporting websocket messages when using "raw" export. + ([#6767](https://github.com/mitmproxy/mitmproxy/pull/6767), @txrp0x9) +* The "save body" feature now also includes WebSocket messages. + ([#6767](https://github.com/mitmproxy/mitmproxy/pull/6767), @txrp0x9) +* Fix compatibility with older cryptography versions and silence a DeprecationWarning on Python <3.11. + ([#6790](https://github.com/mitmproxy/mitmproxy/pull/6790), @mhils) +* Fix a bug when proxying unicode domains. + ([#6796](https://github.com/mitmproxy/mitmproxy/pull/6796), @mhils) + + +## 07 March 2024: mitmproxy 10.2.4 + +* Fix a bug where errors during startup would not be displayed when running mitmproxy. + ([#6719](https://github.com/mitmproxy/mitmproxy/pull/6719), @mhils) +* Use newer cryptography APIs to avoid CryptographyDeprecationWarnings. + This bumps the minimum required version to cryptography 42.0. + ([#6718](https://github.com/mitmproxy/mitmproxy/pull/6718), @mhils) + + +## 06 March 2024: mitmproxy 10.2.3 + +* Fix a regression where `allow_hosts`/`ignore_hosts` would break with IPv6 connections. + ([#6614](https://github.com/mitmproxy/mitmproxy/pull/6614), @dqxpb) +* Fix bug where failed CONNECT request URLs are saved to HAR files incorrectly. + ([#6599](https://github.com/mitmproxy/mitmproxy/pull/6599), @basedBaba) +* Add an arm64 variant for the precompiled macOS app. + ([#6633](https://github.com/mitmproxy/mitmproxy/pull/6633), @mhils) +* Fix duplicate answers being returned in DNS queries. + ([#6648](https://github.com/mitmproxymitmproxy/pull/6648), @sujaldev) +* Fix bug where wireguard config is generated with incorrect endpoint when two or more NICs are active. + ([#6659](https://github.com/mitmproxy/mitmproxy/pull/6659), @basedBaba) +* Fix a regression when leaf cert creation would fail with intermediate CAs in `ca_file`. + ([#6666](https://github.com/mitmproxy/mitmproxy/pull/6666), @manselmi) +* Add `content_view_lines_cutoff` option to mitmdump + ([#6692](https://github.com/mitmproxy/mitmproxy/pull/6692), @errorxyz) +* Allow runtime modifications of HTTP flow filters for server replays + ([#6695](https://github.com/mitmproxy/mitmproxy/pull/6695), @errorxyz) +* Fix bug view options menu in case of overflow + ([#6697](https://github.com/mitmproxy/mitmproxy/pull/6697), @lups2000) +* Allow --allow-hosts and --ignore-hosts to work together + ([#6711](https://github.com/mitmproxy/mitmproxy/pull/6711), @dstd) + + +## 21 January 2024: mitmproxy 10.2.2 + +* Fix a regression where clientplayback would break due to eager task execution. + ([#6605](https://github.com/mitmproxy/mitmproxy/pull/6605), @mhils) +* Fix a regression where WebSocket connections would break due to eager task execution. + ([#6609](https://github.com/mitmproxy/mitmproxy/pull/6609), @mhils) +* Fix bug where insecure HTTP requests are saved incorrectly when exporting to HAR files. + ([#6578](https://github.com/mitmproxy/mitmproxy/pull/6578), @DaniElectra) +* `allow_hosts`/`ignore_hosts` option now matches against the full `host:port` string. + ([#6594](https://github.com/mitmproxy/mitmproxy/pull/6594), @LouisAsanaka) + + +## 06 January 2024: mitmproxy 10.2.1 + +* Fix a regression introduced in mitmproxy 10.2.0: WireGuard servers + now bind to all interfaces again. + ([#6587](https://github.com/mitmproxy/mitmproxy/pull/6587), @mhils) +* Remove stale reference to `ctx.log` in addon documentation. + ([#6552](https://github.com/mitmproxy/mitmproxy/pull/6552), @brojonat) +* Fix a bug where a traceback is shown during shutdown. + ([#6581](https://github.com/mitmproxy/mitmproxy/pull/6581), @mhils) + + +## 04 January 2024: mitmproxy 10.2.0 + +* *Local Redirect Mode* is now officially available on + [macOS](https://mitmproxy.org/posts/local-redirect/macos/) + and [Windows](https://mitmproxy.org/posts/local-redirect/windows/). + See the linked blog posts for details. (@emanuele-em, @mhils) +* UDP streams are now backed by a new implementation in `mitmproxy_rs`. + This represents a major API change as UDP traffic is now exposed as streams + instead of a callback for each packet. (@mhils) +* Fix a regression from mitmproxy 10.1.6 where `ignore_hosts` would terminate requests + instead of forwarding them. + ([#6559](https://github.com/mitmproxy/mitmproxy/pull/6559), @mhils) +* `ignore_hosts` now waits for the entire HTTP headers if it suspects the connection to be HTTP. + ([#6559](https://github.com/mitmproxy/mitmproxy/pull/6559), @mhils) + + +## 14 December 2023: mitmproxy 10.1.6 + +* Fix compatibility with Windows Schannel clients, which previously got + confused by CA and leaf certificate sharing the same Subject Key Identifier. + ([#6549](https://github.com/mitmproxy/mitmproxy/pull/6549), @driuba and @mhils) +* Change keybinding for exporting flow from "e" to "x" to avoid conflict with "edit" keybinding. + ([#6225](https://github.com/mitmproxy/mitmproxy/issues/6225), @Llama1412) +* Fix bug where response flows from HAR files had incorrect `content-length` headers + ([#6548](https://github.com/mitmproxy/mitmproxy/pull/6548), @zanieb) +* Improved handling for `allow_hosts`/`ignore_hosts` options in WireGuard mode (#5930). + ([#6513](https://github.com/mitmproxy/mitmproxy/pull/6513), @dsphper) +* Fix a bug where TCP connections were not closed properly. + ([#6543](https://github.com/mitmproxy/mitmproxy/pull/6543), @mhils) +* DNS resolution is now exempted from `ignore_hosts` in WireGuard Mode. + ([#6513](https://github.com/mitmproxy/mitmproxy/pull/6513), @dsphper) +* Fix case sensitivity of URL added to blocklist + ([#6493](https://github.com/mitmproxy/mitmproxy/pull/6493), @emanuele-em) +* Fix a bug where logging was stopped prematurely during shutdown. + ([#6541](https://github.com/mitmproxy/mitmproxy/pull/6541), @mhils) +* For plaintext traffic, `ignore_hosts` now also takes HTTP/1 host headers into account. + ([#6513](https://github.com/mitmproxy/mitmproxy/pull/6513), @dsphper) +* Fix empty cookie attributes being set to `Key=` instead of `Key` + ([#5084](https://github.com/mitmproxy/mitmproxy/pull/5084), @Speedlulu) +* Scripts with relative paths are now loaded relative to the config file and not where the command is ran + ([#4860](https://github.com/mitmproxy/mitmproxy/pull/4860), @Speedlulu) +* Fix `mitmweb` splitter becoming drag and drop. + ([#6492](https://github.com/mitmproxy/mitmproxy/pull/6492), @xBZZZZ) +* Enhance documentation and add alert log messages when stream_large_bodies and modify_body are set + ([#6514](https://github.com/mitmproxy/mitmproxy/pull/6514), @rosydawn6) + +### Breaking Changes +* Subject Alternative Names are now represented as `cryptography.x509.GeneralNames` instead of `list[str]` + across the codebase. This fixes a regression introduced in mitmproxy 10.1.1 related to punycode domain encoding. + ([#6537](https://github.com/mitmproxy/mitmproxy/pull/6537), @mhils) + + +## 14 November 2023: mitmproxy 10.1.5 + +* Remove stray `replay-extra` from CLI status bar. + ([37d62ce](https://github.com/mitmproxy/mitmproxy/commit/37d62ce73ebd57780cff5ecf8b2ee57ec7d8ab30), @mhils) + + +## 13 November 2023: mitmproxy 10.1.4 + +* Fix a hang/freeze in the macOS distributions when doing TLS negotiation. + ([#6480](https://github.com/mitmproxy/mitmproxy/pull/6480), @mhils) +* Update savehar addon to fix creating corrupt har files caused by empty response content + ([#6459](https://github.com/mitmproxy/mitmproxy/pull/6459), @lain3d) +* Update savehar addon to handle scenarios where "path" key in cookie + attrs dict is missing. + ([#6458](https://github.com/mitmproxy/mitmproxy/pull/6458), @pogzyb) +* Add `server_replay_extra` option to serverplayback to define behaviour + when replayable response is missing. + ([#6465](https://github.com/mitmproxy/mitmproxy/pull/6465), @dkarandikar) + + +## 04 November 2023: mitmproxy 10.1.3 + +* Fix a bug introduced in mitmproxy 10.1.2 where mitmweb would fail to establish + a WebSocket connection. Affected users may need to clear their browser cache + or hard-reload mitmweb (Ctrl+Shift+R). + ([#6454](https://github.com/mitmproxy/mitmproxy/pull/6454), @mhils) + + +## 03 November 2023: mitmproxy 10.1.2 + +* Add a raw hex stream contentview. + ([#6389](https://github.com/mitmproxy/mitmproxy/pull/6389), @mhils) +* Add a contentview for DNS-over-HTTPS. + ([#6389](https://github.com/mitmproxy/mitmproxy/pull/6389), @mhils) +* Replaced standalone mitmproxy binaries on macOS with an app bundle + that contains the mitmproxy/mitmweb/mitmdump CLI tools. + This change was necessary to support macOS code signing requirements. + Homebrew remains the recommended installation method. + ([#6447](https://github.com/mitmproxy/mitmproxy/pull/6447), @mhils) +* Fix certificate generation to work with strict mode OpenSSL 3.x clients + ([#6410](https://github.com/mitmproxy/mitmproxy/pull/6410), @mmaxim) +* Fix path() documentation that the return value might include the query string + ([#6412](https://github.com/mitmproxy/mitmproxy/pull/6412), @tddschn) +* mitmproxy now officially supports Python 3.12. + ([#6434](https://github.com/mitmproxy/mitmproxy/pull/6434), @mhils) +* Fix root-relative URLs so that mitmweb can run in subdirectories. + ([#6411](https://github.com/mitmproxy/mitmproxy/pull/6411), @davet2001) +* Add an optional parameter(ldap search filter key) to ProxyAuth-LDAP. + ([#6428](https://github.com/mitmproxy/mitmproxy/pull/6428), @outlaws-bai) +* Fix a regression when using the proxyauth addon with clients that (rightfully) reuse connections. + ([#6432](https://github.com/mitmproxy/mitmproxy/pull/6432), @mhils) + + +## 27 September 2023: mitmproxy 10.1.1 + +* Fix certificate generation for punycode domains. + ([#6382](https://github.com/mitmproxy/mitmproxy/pull/6382), @mhils) +* Fix a bug that would crash mitmweb when opening options. + ([#6386](https://github.com/mitmproxy/mitmproxy/pull/6386), @mhils) + + +## 24 September 2023: mitmproxy 10.1.0 + +* Add support for reading HAR files using the existing flow loading APIs, e.g. `mitmproxy -r example.har`. + ([#6335](https://github.com/mitmproxy/mitmproxy/pull/6335), @stanleygvi) +* Add support for writing HAR files using the `save.har` command and the `hardump` option for mitmdump. + ([#6368](https://github.com/mitmproxy/mitmproxy/pull/6368), @stanleygvi) +* Packaging changes: + - `mitmproxy-rs` does not depend on a protobuf compiler being available anymore, + we're now also providing a working source distribution for all platforms. + - On macOS, `mitmproxy-rs` now depends on `mitmproxy-macos`. We only provide binary wheels for this package because + it contains a code-signed system extension. Building from source requires a valid Apple Developer Id, see CI for + details. + - On Windows, `mitmproxy-rs` now depends on `mitmproxy-windows`. We only provide binary wheels for this package to + simplify our deployment process, see CI for how to build from source. + + ([#6303](https://github.com/mitmproxy/mitmproxy/issues/6303), @mhils) +* Increase maximum dump file size accepted by mitmweb + ([#6373](https://github.com/mitmproxy/mitmproxy/pull/6373), @t-wy) + + +## 04 August 2023: mitmproxy 10.0.0 + +* Add experimental support for HTTP/3 and QUIC. + ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) +* ASGI/WSGI apps can now listen on all ports for a specific hostname. + This makes it simpler to accept both HTTP and HTTPS. + ([#5725](https://github.com/mitmproxy/mitmproxy/pull/5725), @mhils) +* Add `replay.server.add` command for adding flows to server replay buffer + ([#5851](https://github.com/mitmproxy/mitmproxy/pull/5851), @italankin) +* Remove string escaping in raw view. + ([#5470](https://github.com/mitmproxy/mitmproxy/issues/5470), @stephenspol) +* Updating `Request.port` now also updates the Host header if present. + This aligns with `Request.host`, which already does this. + ([#5908](https://github.com/mitmproxy/mitmproxy/pull/5908), @sujaldev) +* Fix editing of multipart HTTP requests from the CLI. + ([#5148](https://github.com/mitmproxy/mitmproxy/issues/5148), @mhils) +* Add documentation on using Magisk module for intercepting traffic in Android production builds. + ([#5924](https://github.com/mitmproxy/mitmproxy/pull/5924), @Jurrie) +* Fix a bug where the direction indicator in the message stream view would be in the wrong direction. + ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) +* Fix a bug where peername would be None in tls_passthrough script, which would make it not working. + ([#5904](https://github.com/mitmproxy/mitmproxy/pull/5904), @truebit) +* the `esc` key can now be used to exit the current view + ([#6087](https://github.com/mitmproxy/mitmproxy/pull/6087), @sujaldev) +* focus-follow shortcut will now work in flow view context too. + ([#6088](https://github.com/mitmproxy/mitmproxy/pull/6088), @sujaldev) +* Fix a bug where a server connection timeout would cause requests to be issued with a wrong SNI in reverse proxy mode. + ([#6148](https://github.com/mitmproxy/mitmproxy/pull/6148), @mhils) +* The `server_replay_nopop` option has been renamed to `server_replay_reuse` to avoid confusing double-negation. + ([#6084](https://github.com/mitmproxy/mitmproxy/issues/6084), @prady0t, @Semnodime) +* Add zstd to valid gRPC encoding schemes. + ([#6188](https://github.com/mitmproxy/mitmproxy/pull/6188), @tsaaristo) +* For reverse proxy directly accessed via IP address, the IP address is now included + as a subject in the generated certificate. + ([#6202](https://github.com/mitmproxy/mitmproxy/pull/6202), @mhils) +* Enable legacy SSL connect when connecting to server if the `ssl_insecure` flag is set. + ([#6281](https://github.com/mitmproxy/mitmproxy/pull/6281), @DurandA) +* Change wording in the [http-reply-from-proxy.py example](https://github.com/mitmproxy/mitmproxy/blob/main/examples/addons/http-reply-from-proxy.py). + ([#6117](https://github.com/mitmproxy/mitmproxy/pull/6117), @Semnodime) +* Added option to specify an elliptic curve for key exchange between mitmproxy <-> server + ([#6170](https://github.com/mitmproxy/mitmproxy/pull/6170), @Mike-Ki-ASD) +* Add "Prettier" code linting tool to mitmweb. + ([#5985](https://github.com/mitmproxy/mitmproxy/pull/5985), @alexgershberg) +* When logging exceptions, provide the entire exception object to log handlers + ([#6295](https://github.com/mitmproxy/mitmproxy/pull/6295), @mhils) +* mitmproxy now requires Python 3.10 or above. + ([#5954](https://github.com/mitmproxy/mitmproxy/pull/5954), @mhils) + +### Breaking Changes + +* The `onboarding_port` option has been removed. The onboarding app now responds + to all requests for the hostname specified in `onboarding_host`. +* `connection.Client` and `connection.Server` now accept keyword arguments only. + This is a breaking change for custom addons that use these classes directly. ## 07 April 2023: mitmproxy 9.0.1 @@ -29,7 +458,7 @@ ([#5414](https://github.com/mitmproxy/mitmproxy/pull/5414), @meitinger) * Add WireGuard mode to enable transparent proxying via WireGuard. ([#5562](https://github.com/mitmproxy/mitmproxy/pull/5562), @decathorpe, @mhils) -* Add DTLS support. +* Add DTLS support. ([#5397](https://github.com/mitmproxy/mitmproxy/pull/5397), @kckeiks). * Add a quick help bar to mitmproxy. ([#5381](https://github.com/mitmproxy/mitmproxy/pull/5381/), [#5652](https://github.com/mitmproxy/mitmproxy/pull/5652), @kckeiks, @mhils). @@ -43,6 +472,12 @@ See [the docs](https://docs.mitmproxy.org/dev/addons-api-changelog/) for details and upgrade instructions. ([#5590](https://github.com/mitmproxy/mitmproxy/pull/5590), @mhils) +### Breaking Changes + + * The `mode` option is now a list of server specs instead of a single spec. + The CLI interface is unaffected, but users may need to update their `config.yaml`. + ([#5393](https://github.com/mitmproxy/mitmproxy/pull/5393), @mhils) + ### Full Changelog * Mitmproxy binaries now ship with Python 3.11. @@ -53,7 +488,7 @@ ([#5623](https://github.com/mitmproxy/mitmproxy/issues/5623), @SapiensAnatis) * Add MQTT content view. ([#5588](https://github.com/mitmproxy/mitmproxy/pull/5588), @nikitastupin, @abbbe) -* Setting `connection_strategy` to `lazy` now also disables early +* Setting `connection_strategy` to `lazy` now also disables early upstream connections to fetch TLS certificate details. ([#5487](https://github.com/mitmproxy/mitmproxy/pull/5487), @mhils) * Fix order of event hooks on startup. @@ -123,7 +558,7 @@ ([#4469](https://github.com/mitmproxy/mitmproxy/issues/4469), @mhils) * Add flatpak support to the browser addon ([#5200](https://github.com/mitmproxy/mitmproxy/issues/5200), @pauloromeira) -* Add example addon to dump contents to files based on a filter expression +* Add example addon to dump contents to files based on a filter expression ([#5190](https://github.com/mitmproxy/mitmproxy/issues/5190), @redraw) * Fix a bug where the wrong SNI is sent to an upstream HTTPS proxy ([#5109](https://github.com/mitmproxy/mitmproxy/issues/5109), @mhils) @@ -133,14 +568,14 @@ ([#5217](https://github.com/mitmproxy/mitmproxy/issues/5217), @randomstuff) * Improve cut addon to better handle binary contents ([#3965](https://github.com/mitmproxy/mitmproxy/issues/3965), @mhils) -* Fix text truncation for full-width characters +* Fix text truncation for full-width characters ([#4278](https://github.com/mitmproxy/mitmproxy/issues/4278), @kjy00302) * Fix mitmweb export copy failed in non-secure domain. ([#5264](https://github.com/mitmproxy/mitmproxy/issues/5264), @Pactortester) * Add example script for manipulating cookies. ([#5278](https://github.com/mitmproxy/mitmproxy/issues/5278), @WillahScott) -* When opening an external viewer for message contents, mailcap files are not considered anymore. - This preempts the upcoming deprecation of Python's `mailcap` module. +* When opening an external viewer for message contents, mailcap files are not considered anymore. + This preempts the upcoming deprecation of Python's `mailcap` module. ([#5297](https://github.com/mitmproxy/mitmproxy/issues/5297), @KORraNpl) * Fix hostname encoding for IDNA domains in upstream mode. ([#5316](https://github.com/mitmproxy/mitmproxy/issues/5316), @nneonneo) @@ -201,7 +636,7 @@ * Add ability to specify custom ports with LDAP authentication (#5068, @demonoidvk) * Add support for rotating saved streams every hour or day (@EndUser509) * Console Improvements on Windows (@mhils) -* Fix processing of `--set` options (#5067, @marwinxxii) +* Fix processing of `--set` options (#5067, @marwinxxii) * Lowercase user-added header names and emit a log message to notify the user when using HTTP/2 (#4746, @mhils) * Exit early if there are errors on startup (#4544, @mhils) * Fixed encoding guessing: only search for meta tags in HTML bodies (##4566, @Prinzhorn) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21e2f12e2f..c1dc150096 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,8 +18,7 @@ forward, please consider contributing in the following areas: ## Development Setup -To get started hacking on mitmproxy, please install a recent version of Python (we require at least Python 3.9). -Then, do the following: +To get started hacking on mitmproxy, please install the latest version of Python and do the following: ##### Linux / macOS @@ -83,7 +82,7 @@ For speedier testing, you can also run [pytest](http://pytest.org/) directly on ```shell cd test/mitmproxy/addons -pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py +pytest --looponfail test_anticache.py ``` Please ensure that all patches are accompanied by matching changes in the test suite. The project tries to maintain 100% @@ -96,7 +95,7 @@ Keeping to a consistent code style throughout the project makes it easier to con We enforce the following check for all PRs: ```shell -tox -e flake8 +tox -e lint ``` If a linting error is detected, the automated pull request checks will fail and block merging. diff --git a/README.md b/README.md index 38ee2f43ea..8640efbbe2 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ The standard MITMProxy documentation follows below. # mitmproxy [](https://github.com/mitmproxy/mitmproxy/actions?query=branch%3Amain) +[](https://app.codacy.com/gh/mitmproxy/mitmproxy/dashboard) +[](https://autofix.ci) [](https://codecov.io/gh/mitmproxy/mitmproxy) [](https://pypi.python.org/pypi/mitmproxy) [](https://pypi.python.org/pypi/mitmproxy) @@ -72,7 +74,7 @@ General information, tutorials, and precompiled binaries can be found on the mit The documentation for mitmproxy is available on our website: [](https://docs.mitmproxy.org/stable/) -[](https://docs.mitmproxy.org/main/) +[](https://docs.mitmproxy.org/dev/) If you have questions on how to use mitmproxy, please use GitHub Discussions! @@ -84,7 +86,3 @@ use GitHub Discussions! As an open source project, mitmproxy welcomes contributions of all forms. [](./CONTRIBUTING.md) - -Also, please feel free to join our developer Slack! - -[](http://slack.mitmproxy.org/) diff --git a/browserup-proxy.schema.json b/browserup-proxy.schema.json index 552df8abfc..7e54d02df7 100644 --- a/browserup-proxy.schema.json +++ b/browserup-proxy.schema.json @@ -25,7 +25,7 @@ "url": "logo.png" }, "title": "BrowserUp MitmProxy", - "version": "1.23" + "version": "1.24" }, "paths": { "/har": { @@ -634,77 +634,6 @@ } } }, - "WebSocketMessage": { - "type": "object", - "required": [ - "type", - "opcode", - "data", - "time" - ], - "properties": { - "type": { - "type": "string" - }, - "opcode": { - "type": "number" - }, - "data": { - "type": "string" - }, - "time": { - "type": "number" - } - } - }, - "Header": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - }, - "comment": { - "type": "string" - } - } - }, - "Action": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "id": { - "type": "string" - }, - "className": { - "type": "string" - }, - "tagName": { - "type": "string" - }, - "xpath": { - "type": "string" - }, - "dataAttributes": { - "type": "string" - }, - "formName": { - "type": "string" - }, - "content": { - "type": "string" - } - }, - "additionalProperties": false - }, "PageTimings": { "type": "object", "required": [ @@ -785,6 +714,28 @@ } } }, + "CustomHarData": { + "type": "object", + "minProperties": 1 + }, + "Header": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "comment": { + "type": "string" + } + } + }, "Har": { "type": "object", "required": [ @@ -863,6 +814,59 @@ } } }, + "WebSocketMessage": { + "type": "object", + "required": [ + "type", + "opcode", + "data", + "time" + ], + "properties": { + "type": { + "type": "string" + }, + "opcode": { + "type": "number" + }, + "data": { + "type": "string" + }, + "time": { + "type": "number" + } + } + }, + "Action": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "className": { + "type": "string" + }, + "tagName": { + "type": "string" + }, + "xpath": { + "type": "string" + }, + "dataAttributes": { + "type": "string" + }, + "formName": { + "type": "string" + }, + "content": { + "type": "string" + } + }, + "additionalProperties": false + }, "HarEntry": { "type": "object", "required": [ @@ -1322,10 +1326,6 @@ } } }, - "CustomHarData": { - "type": "object", - "minProperties": 1 - }, "Page": { "type": "object", "required": [ diff --git a/docs/build.py b/docs/build.py index f393901e93..d8c3e944c2 100755 --- a/docs/build.py +++ b/docs/build.py @@ -3,7 +3,6 @@ import subprocess from pathlib import Path - here = Path(__file__).parent for script in sorted((here / "scripts").glob("*.py")): diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 7d9971b62f..95bd7f111a 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -2,18 +2,30 @@ import contextlib import inspect import textwrap +import typing from pathlib import Path -from mitmproxy import hooks, log, addonmanager -from mitmproxy.proxy import server_hooks, layer -from mitmproxy.proxy.layers import dns, http, modes, tcp, tls, udp, websocket +from mitmproxy import addonmanager +from mitmproxy import hooks +from mitmproxy import log +from mitmproxy.proxy import layer +from mitmproxy.proxy import server_hooks +from mitmproxy.proxy.layers import dns +from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import udp +from mitmproxy.proxy.layers import websocket +from mitmproxy.proxy.layers.http import _hooks as http known = set() def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: all_params = [ - list(inspect.signature(hook.__init__, eval_str=True).parameters.values())[1:] for hook in hooks + list(inspect.signature(hook.__init__, eval_str=True).parameters.values())[1:] + for hook in hooks ] # slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯ @@ -22,15 +34,9 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: for params in all_params: for param in params: try: - mod = inspect.getmodule(param.annotation).__name__ - if mod == "typing": - # this is ugly, but can be removed once we are on Python 3.9+ only - imports.add( - inspect.getmodule(param.annotation.__args__[0]).__name__ - ) - types.add(param.annotation._name) - else: - imports.add(mod) + imports.add(inspect.getmodule(param.annotation).__name__) + for t in typing.get_args(param.annotation): + imports.add(inspect.getmodule(t).__name__) except AttributeError: raise ValueError(f"Missing type annotation: {params}") imports.discard("builtins") @@ -54,7 +60,8 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: raise RuntimeError(f"Already documented: {hook}") known.add(hook.name) doc = inspect.getdoc(hook) - print(f" def {hook.name}({', '.join(str(p) for p in ['self'] + params)}):") + print(f" @staticmethod") + print(f" def {hook.name}({', '.join(str(p) for p in params)}):") print(textwrap.indent(f'"""\n{doc}\n"""', " ")) if params: print( @@ -90,6 +97,7 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: server_hooks.ServerConnectHook, server_hooks.ServerConnectedHook, server_hooks.ServerDisconnectedHook, + server_hooks.ServerConnectErrorHook, ], ) @@ -104,6 +112,8 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: http.HttpErrorHook, http.HttpConnectHook, http.HttpConnectUpstreamHook, + http.HttpConnectedHook, + http.HttpConnectErrorHook, ], ) @@ -139,6 +149,15 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: ], ) + category( + "QUIC", + "", + [ + quic.QuicStartClientHook, + quic.QuicStartServerHook, + ], + ) + category( "TLS", "", diff --git a/docs/scripts/clirecording/clidirector.py b/docs/scripts/clirecording/clidirector.py index db286b2b2a..973ca610ff 100644 --- a/docs/scripts/clirecording/clidirector.py +++ b/docs/scripts/clirecording/clidirector.py @@ -1,11 +1,11 @@ import json -from typing import NamedTuple, Optional - -import libtmux import random import subprocess import threading import time +from typing import NamedTuple + +import libtmux class InstructionSpec(NamedTuple): @@ -73,7 +73,7 @@ def end_session(self) -> None: self.tmux_session.kill_session() def press_key( - self, keys: str, count=1, pause: Optional[float] = None, target=None + self, keys: str, count=1, pause: float | None = None, target=None ) -> None: if pause is None: pause = self.pause_between_keys @@ -96,7 +96,7 @@ def press_key( real_pause += 2 * pause self.pause(real_pause) - def type(self, keys: str, pause: Optional[float] = None, target=None) -> None: + def type(self, keys: str, pause: float | None = None, target=None) -> None: if pause is None: pause = self.pause_between_keys if target is None: @@ -127,7 +127,7 @@ def run_external(self, command: str) -> None: def message( self, msg: str, - duration: Optional[int] = None, + duration: int | None = None, add_instruction: bool = True, instruction_html: str = "", ) -> None: @@ -160,7 +160,7 @@ def close_popup(self, duration: float = 0) -> None: self.tmux_pane.cmd("display-popup", "-C") def instruction( - self, instruction: str, duration: float = 3, time_from: Optional[float] = None + self, instruction: str, duration: float = 3, time_from: float | None = None ) -> None: if time_from is None: time_from = self.current_time diff --git a/docs/scripts/clirecording/record.py b/docs/scripts/clirecording/record.py index 54ba1be2a7..46f4748224 100644 --- a/docs/scripts/clirecording/record.py +++ b/docs/scripts/clirecording/record.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 - -from clidirector import CliDirector import screenplays - +from clidirector import CliDirector if __name__ == "__main__": director = CliDirector() diff --git a/docs/scripts/clirecording/screenplays.py b/docs/scripts/clirecording/screenplays.py index ea871e7a7f..5f916dac1d 100644 --- a/docs/scripts/clirecording/screenplays.py +++ b/docs/scripts/clirecording/screenplays.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - from clidirector import CliDirector diff --git a/docs/scripts/examples.py b/docs/scripts/examples.py index 4dd742d500..953cd1fccf 100755 --- a/docs/scripts/examples.py +++ b/docs/scripts/examples.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import re from pathlib import Path diff --git a/docs/scripts/filters.py b/docs/scripts/filters.py index 32634196a8..a228593082 100755 --- a/docs/scripts/filters.py +++ b/docs/scripts/filters.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 - from mitmproxy import flowfilter - print('