diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..bc3c076f39 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: "monthly" + groups: + "github actions": + patterns: + - "*" + - package-ecosystem: pip + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 10 diff --git a/.github/node-version.txt b/.github/node-version.txt new file mode 100644 index 0000000000..8351c19397 --- /dev/null +++ b/.github/node-version.txt @@ -0,0 +1 @@ +14 diff --git a/.github/python-version.txt b/.github/python-version.txt new file mode 100644 index 0000000000..2c0733315e --- /dev/null +++ b/.github/python-version.txt @@ -0,0 +1 @@ +3.11 diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 0000000000..8ee552c49e --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,30 @@ +name: autofix.ci + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + autofix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: install-pinned/ruff@0e35bc58bd73769469284df9e1f8898daeea8768 + - run: ruff --fix-only . + - run: ruff format . + + - name: Run prettier + run: | + npm ci + npm run prettier + working-directory: web + + - uses: mhils/add-pr-ref-in-changelog@main + + - uses: autofix-ci/action@d3e591514b99d0fca6779455ff8338516663f7cc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af6ac43378..5e7e68e4b3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,66 +1,42 @@ name: CI -on: [ push, pull_request ] +on: + push: + branches: + - '**' + - '!dependabot/**' + pull_request: + merge_group: + workflow_dispatch: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - lint-pr: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - uses: TrueBrain/actions-flake8@c2deca24d388aa5aedd6478332aa9df4600b5eac # v2.1 - # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions - lint-local: - if: github.event_name == 'push' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - run: pip install tox - - run: tox -e flake8 + lint: + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e lint + filename-matching: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - run: pip install tox - - run: tox -e filename_matching + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e filename_matching + mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - run: pip install tox - - run: tox -e mypy + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e mypy + individual-coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - fetch-depth: 0 - - uses: actions/setup-python@v2 - with: - python-version: '3.9' # there's a weird bug on 3.10 where some lines are not counted as covered. - - run: pip install tox - - run: tox -e individual_coverage + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e individual_coverage + test: strategy: fail-fast: false @@ -73,15 +49,15 @@ jobs: - os: macos-latest py: "3.11" - os: ubuntu-latest - py: 3.9 + py: "3.10" runs-on: ${{ matrix.os }} steps: - run: printenv - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} - run: pip install tox @@ -94,62 +70,86 @@ jobs: # run tests with loopback only. We need to sudo for unshare, which means we need an absolute path for tox. sudo unshare --net -- sh -c "ip link set lo up; $(which tox) -e py" if: matrix.os == 'ubuntu-latest' - - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 - # mirrored below and at https://github.com/mitmproxy/mitmproxy/settings/actions + - uses: mhils/better-codecov-action@main with: - file: ./coverage.xml - name: ${{ matrix.os }} + arguments: '--file ./coverage.xml --name ${{ matrix.os }}' build: strategy: fail-fast: false matrix: include: - - image: macos-10.15 + - image: macos-12 platform: macos - image: windows-2019 platform: windows - - image: ubuntu-18.04 # Old Ubuntu version for old glibc + - image: ubuntu-20.04 # Oldest available version so we get oldest glibc possible. platform: linux runs-on: ${{ matrix.image }} env: - CI_BUILD_WHEEL: ${{ matrix.platform == 'linux' }} - CI_BUILD_PYINSTALLER: 1 - CI_BUILD_WININSTALLER: ${{ matrix.platform == 'windows' }} CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version-file: .github/python-version.txt - if: matrix.platform == 'windows' - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: release/installbuilder/setup key: installbuilder - - run: pip install -e .[dev] - - run: python release/cibuild.py build - # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24 - - uses: actions/upload-artifact@v2 + - run: pip install .[dev] # pyinstaller 5.9 does not like pyproject.toml + editable installs. + + # macOS x64. Due to GHA limitations, we are currently building the Apple Silicon app bundle outside of CI. + - if: matrix.platform == 'macos' && github.repository == 'mitmproxy/mitmproxy' + && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) + id: keychain + uses: apple-actions/import-codesign-certs@5565bb656f60c98c8fc515f3444dd8db73545dc2 + with: + keychain: ${{ runner.temp }}/temp + p12-file-base64: ${{ secrets.APPLE_CERTIFICATE }} + p12-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + - if: matrix.platform == 'macos' && github.repository == 'mitmproxy/mitmproxy' + && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) + run: | + python -u release/build.py macos-app \ + --keychain "${{ runner.temp }}/temp.keychain" \ + --team-id "S8XHQB96PW" \ + --apple-id "${{ secrets.APPLE_ID }}" \ + --password "${{ secrets.APPLE_APP_PASSWORD }}" + + # Linux + - if: matrix.platform == 'linux' + run: python -u release/build.py standalone-binaries wheel + + # Windows + - if: matrix.platform == 'windows' + run: python -u release/build.py standalone-binaries + - if: matrix.platform == 'windows' && github.repository == 'mitmproxy/mitmproxy' && + (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/')) + run: python -u release/build.py --dirty installbuilder-installer msix-installer + + - uses: actions/upload-artifact@v3 with: + # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24 name: binaries.${{ matrix.platform }} - path: release/dist + path: | + release/dist test-web-ui: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: persist-credentials: false - - run: git rev-parse --abbrev-ref HEAD - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v4 with: - node-version: '14' + node-version-file: .github/node-version.txt - name: Cache Node.js modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: # npm cache files are stored in `~/.npm` on Linux/macOS path: ~/.npm @@ -161,100 +161,148 @@ jobs: run: npm ci - working-directory: ./web run: npm test - - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 - # mirrored above and at https://github.com/mitmproxy/mitmproxy/settings/actions + - uses: mhils/better-codecov-action@main with: - file: ./web/coverage/coverage-final.json - name: web + arguments: '--file ./web/coverage/coverage-final.json' docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version-file: .github/python-version.txt - run: | wget -q https://github.com/gohugoio/hugo/releases/download/v0.92.1/hugo_extended_0.92.1_Linux-64bit.deb echo "a9440adfd3ecce40089def287dee4e42ffae252ba08c77d1ac575b880a079ce6 hugo_extended_0.92.1_Linux-64bit.deb" | sha256sum -c sudo dpkg -i hugo*.deb - run: pip install -e .[dev] - run: ./docs/build.py - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: docs path: docs/public + # For releases, also build the archive version of the docs. + - if: startsWith(github.ref, 'refs/tags/') + run: ./docs/build.py + env: + DOCS_ARCHIVE: true + - if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v3 + with: + name: docs-archive + path: docs/public + + check: + if: always() + needs: + - lint + - filename-matching + - mypy + - individual-coverage + - test + - build + - test-web-ui + - docs + uses: mhils/workflows/.github/workflows/alls-green.yml@main + with: + jobs: ${{ toJSON(needs) }} # Separate from everything else because slow. build-and-deploy-docker: if: github.repository == 'mitmproxy/mitmproxy' && ( - github.ref == 'refs/heads/main' || - github.ref == 'refs/heads/dockertest' || - startsWith(github.ref, 'refs/tags/') + github.ref == 'refs/heads/main' + || github.ref == 'refs/heads/citest' + || startsWith(github.ref, 'refs/tags/') ) environment: deploy-docker - needs: - - test - - test-web-ui - - build - - docs + needs: check runs-on: ubuntu-latest env: - CI_BUILD_DOCKER: 1 DOCKER_USERNAME: mitmbot DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.11' - - uses: actions/download-artifact@v2 + python-version-file: .github/python-version.txt + - uses: actions/download-artifact@v3 with: name: binaries.linux path: release/dist - - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1.2.0 - - uses: docker/setup-buildx-action@b1f1f719c7cd5364be7c82e366366da322d01f7c # v1.6.0 - - run: pip install -e .[dev] - - run: python release/cibuild.py build - - run: python release/cibuild.py upload + - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1.6.0 + - run: python release/build-and-deploy-docker.py deploy: # This action has access to our AWS keys, so we are extra careful here. # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks. - if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' - environment: deploy - needs: - - test - - test-web-ui - - build - - docs + if: github.repository == 'mitmproxy/mitmproxy' && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) + environment: ${{ (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/')) && 'deploy-release' || 'deploy-snapshot' }} + needs: check runs-on: ubuntu-latest env: - TWINE_USERNAME: mitmproxy + # PyPI and MSFT keys are only available for the deploy-release environment + # The AWS access key for snapshots is scoped to branches/* as well. + TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-west-2 + MSFT_APP_ID: 9NWNDLQMNZD7 + MSFT_TENANT_ID: ${{ secrets.MSFT_TENANT_ID }} + MSFT_CLIENT_ID: ${{ secrets.MSFT_CLIENT_ID }} + MSFT_CLIENT_SECRET: ${{ secrets.MSFT_CLIENT_SECRET }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version-file: .github/python-version.txt - run: sudo apt-get update - - run: sudo apt-get install -y twine awscli - - uses: actions/download-artifact@v2 + - run: sudo apt-get install -y awscli + - if: startsWith(github.ref, 'refs/tags/') + run: sudo apt-get install -y twine + + - uses: actions/download-artifact@v3 + with: + name: docs + path: docs/public + - if: startsWith(github.ref, 'refs/tags/') + uses: actions/download-artifact@v3 + with: + name: docs-archive + path: docs/archive + - uses: actions/download-artifact@v3 with: + name: binaries.windows path: release/dist - - run: mv release/dist/docs docs/public - # move artifacts from their subfolders into release/dist - - run: find release/dist -mindepth 2 -type f -exec mv {} release/dist \; - # and then delete the empty folders - - run: find release/dist -type d -empty -delete + - uses: actions/download-artifact@v3 + with: + name: binaries.linux + path: release/dist + - uses: actions/download-artifact@v3 + with: + name: binaries.macos + path: release/dist + - run: ls docs/public - run: ls release/dist + - run: ./release/deploy.py + + - name: Deploy to Microsoft Store (test flight) + if: github.ref == 'refs/heads/citest' + run: ./release/deploy-microsoft-store.py release/dist/*.msix + env: + MSFT_APP_FLIGHT: 174ca570-8cae-4444-9858-c07293f1f13a + - name: Deploy to Microsoft Store + if: startsWith(github.ref, 'refs/tags/') + run: ./release/deploy-microsoft-store.py release/dist/*.msix diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..69db1f7eed --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,36 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version (major.minor.patch)' + required: true + type: string + skip-branch-status-check: + description: 'Skip CI status check.' + default: false + required: false + type: boolean + +permissions: + actions: write + contents: write + +jobs: + release: + environment: deploy-release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_PUSH_TOKEN }} # this token works to push to the protected main branch. + - uses: actions/setup-node@v4 + with: + node-version-file: .github/node-version.txt + - uses: actions/setup-python@v4 + with: + python-version-file: .github/python-version.txt + - run: ./release/release.py ${{ inputs.version }} ${{ inputs.skip-branch-status-check }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # this token works with the GraphQL API diff --git a/.gitignore b/.gitignore index 786f221855..2a6f5eb373 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ MANIFEST .tox*/ build/ dist/ -mitmproxy/contrib/kaitaistruct/*.ksy +mitmproxy/contrib/kaitaistruct/**/*.ksy .pytest_cache __pycache__ .hypothesis/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e575e51d4..14f237559f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,249 @@ # Release History + + ## Unreleased: mitmproxy next +* Improved handling for `--allow-hosts`/`--ignore-hosts` options in WireGuard mode (#5930). + ([#6513](https://github.com/mitmproxy/mitmproxy/pull/6513), @dsphper) +* DNS resolution is now exempted from `--ignore-hosts` in WireGuard Mode. + ([#6513](https://github.com/mitmproxy/mitmproxy/pull/6513), @dsphper) +* For plaintext traffic, `--ignore-hosts` now also takes HTTP/1 host headers into account. + ([#6513](https://github.com/mitmproxy/mitmproxy/pull/6513), @dsphper) +* Fix empty cookie attributes being set to `Key=` instead of `Key` + ([#5084](https://github.com/mitmproxy/mitmproxy/pull/5084), @Speedlulu) +* Scripts with relative paths are now loaded relative to the config file and not where the command is ran + ([#4860](https://github.com/mitmproxy/mitmproxy/pull/4860), @Speedlulu) +* Enhance documentation and add alert log messages when stream_large_bodies and modify_body are set + ([#6514](https://github.com/mitmproxy/mitmproxy/pull/6514), @rosydawn6) + + +## 14 November 2023: mitmproxy 10.1.5 + +* Remove stray `replay-extra` from CLI status bar. + ([37d62ce](https://github.com/mitmproxy/mitmproxy/commit/37d62ce73ebd57780cff5ecf8b2ee57ec7d8ab30), @mhils) + + +## 13 November 2023: mitmproxy 10.1.4 + +* Fix a hang/freeze in the macOS distributions when doing TLS negotiation. + ([#6480](https://github.com/mitmproxy/mitmproxy/pull/6480), @mhils) +* Update savehar addon to fix creating corrupt har files caused by empty response content + ([#6459](https://github.com/mitmproxy/mitmproxy/pull/6459), @lain3d) +* Update savehar addon to handle scenarios where "path" key in cookie + attrs dict is missing. + ([#6458](https://github.com/mitmproxy/mitmproxy/pull/6458), @pogzyb) +* Add `server_replay_extra` option to serverplayback to define behaviour + when replayable response is missing. + ([#6465](https://github.com/mitmproxy/mitmproxy/pull/6465), @dkarandikar) + + +## 04 November 2023: mitmproxy 10.1.3 + +* Fix a bug introduced in mitmproxy 10.1.2 where mitmweb would fail to establish + a WebSocket connection. Affected users may need to clear their browser cache + or hard-reload mitmweb (Ctrl+Shift+R). + ([#6454](https://github.com/mitmproxy/mitmproxy/pull/6454), @mhils) + + +## 03 November 2023: mitmproxy 10.1.2 + +* Add a raw hex stream contentview. + ([#6389](https://github.com/mitmproxy/mitmproxy/pull/6389), @mhils) +* Add a contentview for DNS-over-HTTPS. + ([#6389](https://github.com/mitmproxy/mitmproxy/pull/6389), @mhils) +* Replaced standalone mitmproxy binaries on macOS with an app bundle + that contains the mitmproxy/mitmweb/mitmdump CLI tools. + This change was necessary to support macOS code signing requirements. + Homebrew remains the recommended installation method. + ([#6447](https://github.com/mitmproxy/mitmproxy/pull/6447), @mhils) +* Fix certificate generation to work with strict mode OpenSSL 3.x clients + ([#6410](https://github.com/mitmproxy/mitmproxy/pull/6410), @mmaxim) +* Fix path() documentation that the return value might include the query string + ([#6412](https://github.com/mitmproxy/mitmproxy/pull/6412), @tddschn) +* mitmproxy now officially supports Python 3.12. + ([#6434](https://github.com/mitmproxy/mitmproxy/pull/6434), @mhils) +* Fix root-relative URLs so that mitmweb can run in subdirectories. + ([#6411](https://github.com/mitmproxy/mitmproxy/pull/6411), @davet2001) +* Add an optional parameter(ldap search filter key) to ProxyAuth-LDAP. + ([#6428](https://github.com/mitmproxy/mitmproxy/pull/6428), @outlaws-bai) +* Fix a regression when using the proxyauth addon with clients that (rightfully) reuse connections. + ([#6432](https://github.com/mitmproxy/mitmproxy/pull/6432), @mhils) + + +## 27 September 2023: mitmproxy 10.1.1 + +* Fix certificate generation for punycode domains. + ([#6382](https://github.com/mitmproxy/mitmproxy/pull/6382), @mhils) +* Fix a bug that would crash mitmweb when opening options. + ([#6386](https://github.com/mitmproxy/mitmproxy/pull/6386), @mhils) + + +## 24 September 2023: mitmproxy 10.1.0 + +* Add support for reading HAR files using the existing flow loading APIs, e.g. `mitmproxy -r example.har`. + ([#6335](https://github.com/mitmproxy/mitmproxy/pull/6335), @stanleygvi) +* Add support for writing HAR files using the `save.har` command and the `hardump` option for mitmdump. + ([#6368](https://github.com/mitmproxy/mitmproxy/pull/6368), @stanleygvi) +* Packaging changes: + - `mitmproxy-rs` does not depend on a protobuf compiler being available anymore, + we're now also providing a working source distribution for all platforms. + - On macOS, `mitmproxy-rs` now depends on `mitmproxy-macos`. We only provide binary wheels for this package because + it contains a code-signed system extension. Building from source requires a valid Apple Developer Id, see CI for + details. + - On Windows, `mitmproxy-rs` now depends on `mitmproxy-windows`. We only provide binary wheels for this package to + simplify our deployment process, see CI for how to build from source. + + ([#6303](https://github.com/mitmproxy/mitmproxy/issues/6303), @mhils) +* Increase maximum dump file size accepted by mitmweb + ([#6373](https://github.com/mitmproxy/mitmproxy/pull/6373), @t-wy) + + +## 04 August 2023: mitmproxy 10.0.0 + +* Add experimental support for HTTP/3 and QUIC. + ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) +* ASGI/WSGI apps can now listen on all ports for a specific hostname. + This makes it simpler to accept both HTTP and HTTPS. + ([#5725](https://github.com/mitmproxy/mitmproxy/pull/5725), @mhils) +* Add `replay.server.add` command for adding flows to server replay buffer + ([#5851](https://github.com/mitmproxy/mitmproxy/pull/5851), @italankin) +* Remove string escaping in raw view. + ([#5470](https://github.com/mitmproxy/mitmproxy/issues/5470), @stephenspol) +* Updating `Request.port` now also updates the Host header if present. + This aligns with `Request.host`, which already does this. + ([#5908](https://github.com/mitmproxy/mitmproxy/pull/5908), @sujaldev) +* Fix editing of multipart HTTP requests from the CLI. + ([#5148](https://github.com/mitmproxy/mitmproxy/issues/5148), @mhils) +* Add documentation on using Magisk module for intercepting traffic in Android production builds. + ([#5924](https://github.com/mitmproxy/mitmproxy/pull/5924), @Jurrie) +* Fix a bug where the direction indicator in the message stream view would be in the wrong direction. + ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) +* Fix a bug where peername would be None in tls_passthrough script, which would make it not working. + ([#5904](https://github.com/mitmproxy/mitmproxy/pull/5904), @truebit) +* the `esc` key can now be used to exit the current view + ([#6087](https://github.com/mitmproxy/mitmproxy/pull/6087), @sujaldev) +* focus-follow shortcut will now work in flow view context too. + ([#6088](https://github.com/mitmproxy/mitmproxy/pull/6088), @sujaldev) +* Fix a bug where a server connection timeout would cause requests to be issued with a wrong SNI in reverse proxy mode. + ([#6148](https://github.com/mitmproxy/mitmproxy/pull/6148), @mhils) +* The `server_replay_nopop` option has been renamed to `server_replay_reuse` to avoid confusing double-negation. + ([#6084](https://github.com/mitmproxy/mitmproxy/issues/6084), @prady0t, @Semnodime) +* Add zstd to valid gRPC encoding schemes. + ([#6188](https://github.com/mitmproxy/mitmproxy/pull/6188), @tsaaristo) +* For reverse proxy directly accessed via IP address, the IP address is now included + as a subject in the generated certificate. + ([#6202](https://github.com/mitmproxy/mitmproxy/pull/6202), @mhils) +* Enable legacy SSL connect when connecting to server if the `ssl_insecure` flag is set. + ([#6281](https://github.com/mitmproxy/mitmproxy/pull/6281), @DurandA) +* Change wording in the [http-reply-from-proxy.py example](https://github.com/mitmproxy/mitmproxy/blob/main/examples/addons/http-reply-from-proxy.py). + ([#6117](https://github.com/mitmproxy/mitmproxy/pull/6117), @Semnodime) +* Added option to specify an elliptic curve for key exchange between mitmproxy <-> server + ([#6170](https://github.com/mitmproxy/mitmproxy/pull/6170), @Mike-Ki-ASD) +* Add "Prettier" code linting tool to mitmweb. + ([#5985](https://github.com/mitmproxy/mitmproxy/pull/5985), @alexgershberg) +* When logging exceptions, provide the entire exception object to log handlers + ([#6295](https://github.com/mitmproxy/mitmproxy/pull/6295), @mhils) +* mitmproxy now requires Python 3.10 or above. + ([#5954](https://github.com/mitmproxy/mitmproxy/pull/5954), @mhils) + +### Breaking Changes + +* The `onboarding_port` option has been removed. The onboarding app now responds + to all requests for the hostname specified in `onboarding_host`. +* `connection.Client` and `connection.Server` now accept keyword arguments only. + This is a breaking change for custom addons that use these classes directly. + +## 02 November 2022: mitmproxy 9.0.1 + +* The precompiled binaries now ship with OpenSSL 3.0.7, which resolves CVE-2022-3602 and CVE-2022-3786. +* Performance and stability improvements for WireGuard mode. + ([#5694](https://github.com/mitmproxy/mitmproxy/issues/5694), @mhils, @decathorpe) +* Fix a bug where the standalone Linux binaries would require libffi to be installed. + ([#5699](https://github.com/mitmproxy/mitmproxy/issues/5699), @mhils) +* Hard exit when mitmproxy cannot write logs, fixes endless loop when parent process exits. + ([#4669](https://github.com/mitmproxy/mitmproxy/issues/4669), @Prinzhorn) +* Fix a permission error affecting the Docker images. + ([#5700](https://github.com/mitmproxy/mitmproxy/issues/5700), @mhils) + + +## 28 October 2022: mitmproxy 9.0.0 + +### Major Features + +* Add Raw UDP support. + ([#5414](https://github.com/mitmproxy/mitmproxy/pull/5414), @meitinger) +* Add WireGuard mode to enable transparent proxying via WireGuard. + ([#5562](https://github.com/mitmproxy/mitmproxy/pull/5562), @decathorpe, @mhils) +* Add DTLS support. + ([#5397](https://github.com/mitmproxy/mitmproxy/pull/5397), @kckeiks). +* Add a quick help bar to mitmproxy. + ([#5381](https://github.com/mitmproxy/mitmproxy/pull/5381/), [#5652](https://github.com/mitmproxy/mitmproxy/pull/5652), @kckeiks, @mhils). + +### Deprecations + +* Deprecate `add_log` event hook. Users should use the builtin `logging` module instead. + See [the docs](https://docs.mitmproxy.org/dev/addons-api-changelog/) for details and upgrade instructions. + ([#5590](https://github.com/mitmproxy/mitmproxy/pull/5590), @mhils) +* Deprecate `mitmproxy.ctx.log` in favor of Python's builtin `logging` module. + See [the docs](https://docs.mitmproxy.org/dev/addons-api-changelog/) for details and upgrade instructions. + ([#5590](https://github.com/mitmproxy/mitmproxy/pull/5590), @mhils) + +### Breaking Changes + + * The `mode` option is now a list of server specs instead of a single spec. + The CLI interface is unaffected, but users may need to update their `config.yaml`. + ([#5393](https://github.com/mitmproxy/mitmproxy/pull/5393), @mhils) + +### Full Changelog + +* Mitmproxy binaries now ship with Python 3.11. + ([#5678](https://github.com/mitmproxy/mitmproxy/issues/5678), @mhils) +* One mitmproxy instance can now spawn multiple proxy servers. + ([#5393](https://github.com/mitmproxy/mitmproxy/pull/5393), @mhils) +* Add syntax highlighting to JSON and msgpack content view. + ([#5623](https://github.com/mitmproxy/mitmproxy/issues/5623), @SapiensAnatis) +* Add MQTT content view. + ([#5588](https://github.com/mitmproxy/mitmproxy/pull/5588), @nikitastupin, @abbbe) +* Setting `connection_strategy` to `lazy` now also disables early + upstream connections to fetch TLS certificate details. + ([#5487](https://github.com/mitmproxy/mitmproxy/pull/5487), @mhils) +* Fix order of event hooks on startup. + ([#5376](https://github.com/mitmproxy/mitmproxy/issues/5376), @meitinger) +* Include server information in bind/listen errors. + ([#5495](https://github.com/mitmproxy/mitmproxy/pull/5495), @meitinger) +* Include information about lazy connection_strategy in related errors. + ([#5465](https://github.com/mitmproxy/mitmproxy/pull/5465), @meitinger, @mhils) +* Fix `tls_version_server_min` and `tls_version_server_max` options. + ([#5546](https://github.com/mitmproxy/mitmproxy/issues/5546), @mhils) +* Added Magisk module generation for Android onboarding. + ([#5547](https://github.com/mitmproxy/mitmproxy/pull/5547), @jorants) +* Update Linux binary builder to Ubuntu 20.04, bumping the minimum glibc version to 2.31. + ([#5547](https://github.com/mitmproxy/mitmproxy/pull/5547), @jorants) +* Add "Save filtered" button in mitmweb. + ([#5531](https://github.com/mitmproxy/mitmproxy/pull/5531), @rnbwdsh, @mhils) +* Render application/prpc content as gRPC/Protocol Buffers + ([#5568](https://github.com/mitmproxy/mitmproxy/pull/5568), @selfisekai) +* Mitmweb now supports `content_view_lines_cutoff`. + ([#5548](https://github.com/mitmproxy/mitmproxy/pull/5548), @sanlengjingvv) +* Fix a mitmweb crash when scrolling down the flow list. + ([#5507](https://github.com/mitmproxy/mitmproxy/pull/5507), @LIU-shuyi) +* Add HTTP/3 binary frame content view. + ([#5582](https://github.com/mitmproxy/mitmproxy/pull/5582), @mhils) +* Fix mitmweb not properly opening a browser and being stuck on some Linux. + ([#5522](https://github.com/mitmproxy/mitmproxy/issues/5522), @Prinzhorn) +* Fix race condition when updating mitmweb WebSocket connections that are closing. + ([#5405](https://github.com/mitmproxy/mitmproxy/issues/5405), [#5686](https://github.com/mitmproxy/mitmproxy/issues/5686), @mhils) +* Fix mitmweb crash when using filters. + ([#5658](https://github.com/mitmproxy/mitmproxy/issues/5658), [#5661](https://github.com/mitmproxy/mitmproxy/issues/5661), @LIU-shuyi, @mhils) +* Fix missing default port when starting a browser. + ([#5687](https://github.com/mitmproxy/mitmproxy/issues/5687), @rbdixon) +* Add docs for transparent mode on Windows. + ([#5402](https://github.com/mitmproxy/mitmproxy/issues/5402), @stephenspol) + ## 28 June 2022: mitmproxy 8.1.1 * Support specifying the local address for outgoing connections @@ -15,6 +257,8 @@ * Remove overambitious assertions in the HTTP state machine, fix some error handling. ([#5383](https://github.com/mitmproxy/mitmproxy/issues/5383), @mhils) +* Use default_factory for parser_options. + ([#5474](https://github.com/mitmproxy/mitmproxy/issues/5474), @rathann) ## 15 May 2022: mitmproxy 8.1.0 @@ -63,6 +307,8 @@ ([#5339](https://github.com/mitmproxy/mitmproxy/issues/5339), @mhils) * Fix handling of multiple Cookie headers when proxying HTTP/2 to HTTP/1 ([#5337](https://github.com/mitmproxy/mitmproxy/issues/5337), @rinsuki) +* Improve http_manipulate_cookies.py example. + ([#5578](https://github.com/mitmproxy/mitmproxy/issues/5578), @insilications) ## 19 March 2022: mitmproxy 8.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f0496090a..6a421768a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,7 @@ forward, please consider contributing in the following areas: ## Development Setup -To get started hacking on mitmproxy, please install a recent version of Python (we require at least Python 3.9). -Then, do the following: +To get started hacking on mitmproxy, please install the latest version of Python and do the following: ##### Linux / macOS @@ -79,7 +78,7 @@ For speedier testing, you can also run [pytest](http://pytest.org/) directly on ```shell cd test/mitmproxy/addons -pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py +pytest --looponfail test_anticache.py ``` Please ensure that all patches are accompanied by matching changes in the test suite. The project tries to maintain 100% @@ -92,7 +91,7 @@ Keeping to a consistent code style throughout the project makes it easier to con We enforce the following check for all PRs: ```shell -tox -e flake8 +tox -e lint ``` If a linting error is detected, the automated pull request checks will fail and block merging. diff --git a/README.md b/README.md index a92e0b0b76..f23dbd7c9e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # mitmproxy [](https://github.com/mitmproxy/mitmproxy/actions?query=branch%3Amain) +[](https://app.codacy.com/gh/mitmproxy/mitmproxy/dashboard) +[](https://autofix.ci) [](https://codecov.io/gh/mitmproxy/mitmproxy) [](https://pypi.python.org/pypi/mitmproxy) [](https://pypi.python.org/pypi/mitmproxy) @@ -26,7 +28,7 @@ General information, tutorials, and precompiled binaries can be found on the mit The documentation for mitmproxy is available on our website: [](https://docs.mitmproxy.org/stable/) -[](https://docs.mitmproxy.org/main/) +[](https://docs.mitmproxy.org/dev/) If you have questions on how to use mitmproxy, please use GitHub Discussions! @@ -39,6 +41,6 @@ As an open source project, mitmproxy welcomes contributions of all forms. [](./CONTRIBUTING.md) -Also, please feel free to join our developer Slack! +Also, please feel free to join our developer Slack! However, please note that the primary purpose of our Slack is direct communication between maintainers and contributors. **If you have questions where the answer might be valuable to others, please use [GitHub Discussions](https://github.com/mitmproxy/mitmproxy/discussions) and not Slack.** [](http://slack.mitmproxy.org/) diff --git a/codecov.yml b/codecov.yml index 11f75c939d..899cd7e308 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,7 +7,8 @@ coverage: target: auto threshold: 0.1% paths: - - "!web/" + - "mitmproxy/" + - "examples/addons/" web: target: auto threshold: 0.1% diff --git a/docs/build.py b/docs/build.py index f393901e93..d8c3e944c2 100755 --- a/docs/build.py +++ b/docs/build.py @@ -3,7 +3,6 @@ import subprocess from pathlib import Path - here = Path(__file__).parent for script in sorted((here / "scripts").glob("*.py")): diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 0c23fbd86c..be3f1e0b04 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -2,18 +2,30 @@ import contextlib import inspect import textwrap +import typing from pathlib import Path -from mitmproxy import hooks, log, addonmanager -from mitmproxy.proxy import server_hooks, layer -from mitmproxy.proxy.layers import dns, http, modes, tcp, tls, websocket +from mitmproxy import addonmanager +from mitmproxy import hooks +from mitmproxy import log +from mitmproxy.proxy import layer +from mitmproxy.proxy import server_hooks +from mitmproxy.proxy.layers import dns +from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import udp +from mitmproxy.proxy.layers import websocket +from mitmproxy.proxy.layers.http import _hooks as http known = set() def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: all_params = [ - list(inspect.signature(hook.__init__).parameters.values())[1:] for hook in hooks + list(inspect.signature(hook.__init__, eval_str=True).parameters.values())[1:] + for hook in hooks ] # slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯ @@ -22,21 +34,15 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: for params in all_params: for param in params: try: - mod = inspect.getmodule(param.annotation).__name__ - if mod == "typing": - # this is ugly, but can be removed once we are on Python 3.9+ only - imports.add( - inspect.getmodule(param.annotation.__args__[0]).__name__ - ) - types.add(param.annotation._name) - else: - imports.add(mod) + imports.add(inspect.getmodule(param.annotation).__name__) + for t in typing.get_args(param.annotation): + imports.add(inspect.getmodule(t).__name__) except AttributeError: raise ValueError(f"Missing type annotation: {params}") imports.discard("builtins") if types: print(f"from typing import {', '.join(sorted(types))}") - print("from mitmproxy import ctx") + print("import logging") for imp in sorted(imports): print(f"import {imp}") print() @@ -54,14 +60,15 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: raise RuntimeError(f"Already documented: {hook}") known.add(hook.name) doc = inspect.getdoc(hook) - print(f" def {hook.name}({', '.join(str(p) for p in ['self'] + params)}):") + print(f" @staticmethod") + print(f" def {hook.name}({', '.join(str(p) for p in params)}):") print(textwrap.indent(f'"""\n{doc}\n"""', " ")) if params: print( - f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")' + f' logging.info(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")' ) else: - print(f' ctx.log("{hook.name}")') + print(f' logging.info("{hook.name}")') print("") @@ -128,6 +135,26 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: ], ) + category( + "UDP", + "", + [ + udp.UdpStartHook, + udp.UdpMessageHook, + udp.UdpEndHook, + udp.UdpErrorHook, + ], + ) + + category( + "QUIC", + "", + [ + quic.QuicStartClientHook, + quic.QuicStartServerHook, + ], + ) + category( "TLS", "", diff --git a/docs/scripts/api-render.py b/docs/scripts/api-render.py index 10299e7c3c..0a224eb77f 100644 --- a/docs/scripts/api-render.py +++ b/docs/scripts/api-render.py @@ -32,9 +32,11 @@ "mitmproxy.http", "mitmproxy.net.server_spec", "mitmproxy.proxy.context", + "mitmproxy.proxy.mode_specs", "mitmproxy.proxy.server_hooks", "mitmproxy.tcp", "mitmproxy.tls", + "mitmproxy.udp", "mitmproxy.websocket", here / ".." / "src" / "generated" / "events.py", ] @@ -51,7 +53,7 @@ if isinstance(module, Path): continue filename = f"api/{module.replace('.', '/')}.html" - (api_content / f"{module}.md").write_text( + (api_content / f"{module}.md").write_bytes( textwrap.dedent( f""" --- @@ -65,7 +67,7 @@ {{{{< readfile file="/generated/{filename}" >}}}} """ - ) + ).encode() ) (here / ".." / "src" / "content" / "addons-api.md").touch() diff --git a/docs/scripts/clirecording/clidirector.py b/docs/scripts/clirecording/clidirector.py index db286b2b2a..973ca610ff 100644 --- a/docs/scripts/clirecording/clidirector.py +++ b/docs/scripts/clirecording/clidirector.py @@ -1,11 +1,11 @@ import json -from typing import NamedTuple, Optional - -import libtmux import random import subprocess import threading import time +from typing import NamedTuple + +import libtmux class InstructionSpec(NamedTuple): @@ -73,7 +73,7 @@ def end_session(self) -> None: self.tmux_session.kill_session() def press_key( - self, keys: str, count=1, pause: Optional[float] = None, target=None + self, keys: str, count=1, pause: float | None = None, target=None ) -> None: if pause is None: pause = self.pause_between_keys @@ -96,7 +96,7 @@ def press_key( real_pause += 2 * pause self.pause(real_pause) - def type(self, keys: str, pause: Optional[float] = None, target=None) -> None: + def type(self, keys: str, pause: float | None = None, target=None) -> None: if pause is None: pause = self.pause_between_keys if target is None: @@ -127,7 +127,7 @@ def run_external(self, command: str) -> None: def message( self, msg: str, - duration: Optional[int] = None, + duration: int | None = None, add_instruction: bool = True, instruction_html: str = "", ) -> None: @@ -160,7 +160,7 @@ def close_popup(self, duration: float = 0) -> None: self.tmux_pane.cmd("display-popup", "-C") def instruction( - self, instruction: str, duration: float = 3, time_from: Optional[float] = None + self, instruction: str, duration: float = 3, time_from: float | None = None ) -> None: if time_from is None: time_from = self.current_time diff --git a/docs/scripts/clirecording/record.py b/docs/scripts/clirecording/record.py index 54ba1be2a7..46f4748224 100644 --- a/docs/scripts/clirecording/record.py +++ b/docs/scripts/clirecording/record.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 - -from clidirector import CliDirector import screenplays - +from clidirector import CliDirector if __name__ == "__main__": director = CliDirector() diff --git a/docs/scripts/clirecording/screenplays.py b/docs/scripts/clirecording/screenplays.py index ea871e7a7f..5f916dac1d 100644 --- a/docs/scripts/clirecording/screenplays.py +++ b/docs/scripts/clirecording/screenplays.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - from clidirector import CliDirector diff --git a/docs/scripts/examples.py b/docs/scripts/examples.py index 4dd742d500..953cd1fccf 100755 --- a/docs/scripts/examples.py +++ b/docs/scripts/examples.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import re from pathlib import Path diff --git a/docs/scripts/filters.py b/docs/scripts/filters.py index 32634196a8..a228593082 100755 --- a/docs/scripts/filters.py +++ b/docs/scripts/filters.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 - from mitmproxy import flowfilter - print('