From f989889ac94da9ad00a27fddbb8526ab64a07386 Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 18:28:21 +0100 Subject: [PATCH 1/8] chore: release 2.0.0 - Drop Python 3.8 and 3.9 support; require Python >=3.10 - Add Python 3.13 and 3.14 support to classifiers and CI matrix - Allow `ProtoModule(file_path=None)`: auto-generate unique filename (`definition_N.proto`) - Fix `KeyError` in `test_compile_simple_dependency` by deduplicating `ProtoCollection.modules` values with `set()` - Migrate tox config from `tox.ini` into `pyproject.toml`; adopt `tox-uv` - Migrate CI from `actions/setup-python` + pip to `astral-sh/setup-uv` - Update GitHub Actions plugins to latest: `checkout@v6`, `setup-uv@v7`, `setup-python@v6` - Add `ty` type-checker to lint tox environment; remove Black - Fix README wording: replace "a `.proto` string" with "a protobuf definition string" - Add CHANGELOG.md covering all releases from 0.0.2 to 2.0.0 --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 119 ++------ .pre-commit-config.yaml | 24 +- CHANGELOG.md | 278 ++++++++++++++++++ CONTRIBUTING.md | 95 +++--- LICENSE | 2 +- README.md | 87 +++--- TESTING.md | 61 ++++ pyproject.toml | 150 ++++++++-- setup.py | 5 +- src/proto_topy.py | 75 ++--- .../test_google_addressbook_example.yaml | 50 ++-- tests/test_proto_topy.py | 125 +++++++- tests/tox_mac.sh | 5 +- tox.ini | 64 ---- 15 files changed, 782 insertions(+), 360 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 TESTING.md delete mode 100644 tox.ini diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c22504e..8c9de12 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 - name: Install tools diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2317e18..d556e59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,125 +2,42 @@ name: test on: push: pull_request: + workflow_dispatch: schedule: - - cron: "0 8 * * *" + - cron: "0 8 * * 1" # every Monday at 8am UTC jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e '.[dev]' - - name: Format with Black - run: black . - - name: Lint with Ruff - run: ruff check . - - - py_39_proto_203: - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest, macos-13, windows-latest ] - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - architecture: x64 - cache: pip - - name: Install Protoc - uses: arduino/setup-protoc@v1.1.2 - with: - version: 3.20.3 - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: install dependencies - run: | - pip install --upgrade pip - pip install -e '.[ci]' - protoc --version - pip freeze - - name: Setup test suite - run: tox -vv --notest - - name: Run test suite - run: tox --skip-pkg-install + - uses: astral-sh/setup-uv@v7 + - run: uv run --extra dev ruff check . + - run: uv run --extra dev ty check - py_31x_proto_252: + test: strategy: fail-fast: false matrix: - # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs - os: [ ubuntu-latest, macos-13, windows-latest ] - python: [ '3.10', '3.11'] + # https://docs.github.com/en/actions/using-jobs/using-a-matrix + os: [ubuntu-latest, macos-latest, windows-latest] + protoc: ["25.x", "34.x"] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v4 # https://github.com/actions/checkout + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 - # https://github.com/actions/setup-python - with: - python-version: ${{ matrix.python }} - architecture: x64 - cache: pip - - name: Install Protoc - uses: arduino/setup-protoc@v3 + # https://github.com/astral-sh/setup-uv # https://github.com/arduino/setup-protoc + - uses: astral-sh/setup-uv@v7 + - uses: arduino/setup-protoc@v3 with: - version: 25.2 + version: ${{ matrix.protoc }} repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: install dependencies - run: | - pip install --upgrade pip - pip install -e '.[ci]' - protoc --version - pip freeze - - name: Setup test suite - run: tox -vv --notest - - name: Run test suite - run: tox --skip-pkg-install - - py_3x_proto_25x: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 + - run: uv run --extra ci tox -e py310,py311,py312,py313,py314 + - uses: codecov/codecov-action@v5 + if: matrix.os == 'ubuntu-latest' && matrix.protoc == '25.x' with: - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - architecture: x64 - cache: pip - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - version: 25.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: install dependencies - run: | - pip install --upgrade pip - pip install -e '.[ci]' - pip --version - pip freeze - - name: Setup test suite - run: tox -vv --notest - - name: Run test suite - run: tox --skip-pkg-install + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6afc2b..a17e21f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,30 +1,16 @@ # To install the git pre-commit hook run: # pre-commit install # To update the pre-commit hooks run: -# pre-commit install-hooks +# pre-commit autoupdate exclude: '^(\.tox|\.bumpversion\.cfg)(/|$)' repos: - - repo: https://github.com/psf/black - rev: 24.1.1 - hooks: - - id: black - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.6.2" + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.11.2" hooks: - id: ruff - - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 - hooks: - - id: pyupgrade - args: [ '--py38-plus' ] - - repo: https://github.com/mgedmin/check-python-versions - rev: 0.22.0 - hooks: - - id: check-python-versions - args: [ '--only', 'pyproject.toml,tox.ini' ] - verbose: true + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8406288 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,278 @@ +# Changelog + +## [2.0.0] - 2026-03-15 + +### Added +- `ProtoModule` now accepts `file_path=None`: a unique filename (`definition_N.proto`) is auto-generated via a class-level counter. +- Python 3.13 and 3.14 support added to classifiers and CI matrix. +- `ty` type-checker added to the lint tox environment. +- `ruff` isort configuration added (`combine-as-imports = true`). + +### Fixed +- `KeyError` in `test_compile_simple_dependency` caused by iterating duplicate `ProtoModule` values in `ProtoCollection.modules`. Fixed by deduplicating with `set()` before iterating. +- `compiler_version()` now correctly returns `None` instead of raising when no output is produced. + +### Changed +- **Breaking**: dropped Python 3.8 and 3.9 support; minimum is now Python 3.10. +- `requires-python` updated from `>=3.8` to `>=3.10`. +- `ProtoCollection.modules` now indexes each module by both `Path` and `str` key for flexible lookup. +- Tox configuration migrated from `tox.ini` into `pyproject.toml`; `tox-uv` added as a requirement. +- `urllib3` version constraint relaxed (removed `<2` pin). +- CI: removed `py_39_proto_203` job; `py_31x_proto_252` matrix extended to include Python 3.12, 3.13, 3.14. +- CI: migrated `test.yml` from `actions/setup-python` + pip to `astral-sh/setup-uv`; replaced `pip install` steps with `uv run --extra`; removed Black from lint job. +- CI: updated GitHub Actions plugins to latest versions: `actions/checkout@v6`, `astral-sh/setup-uv@v7`, `actions/setup-python@v6`. + +--- + +## [1.0.5] - 2024-08-29 + +### Added +- `workflow_dispatch` trigger added to CI workflow. +- `CONTRIBUTING.md` enhanced with releasing details. + +### Changed +- Update versions of pre-commit hooks; use `macos-13` where `macos-14` is not supported. +- Exclude `setup.py` from `check-python-versions`; relax the check in the generated `.py`. + +--- + +## [1.0.4] - 2024-02-07 + +### Fixed +- Repair wheel build. + +### Changed +- Reintroduce `tox.ini`. +- Remove tagging from `bumpver` config. +- Adapt `CONTRIBUTING.md` to the releasing process. + +--- + +## [1.0.3] - 2024-02-06 + +### Added +- Add a release workflow. + +--- + +## [1.0.2] - 2024-02-06 + +### Added +- Add `README.md` to `pyproject.toml`. +- Add more details to `CONTRIBUTING.md`. +- Add Black dependency. + +### Changed +- Move CI packages dependency to `pyproject.toml`. + +--- + +## [1.0.1] - 2024-02-05 + +### Changed +- Remove direct dependency on Black in GitHub Actions. +- Isolate `compiler_path` to `compiled()` methods. + +--- + +## [1.0.0] - 2024-02-04 + +### Added +- `ruff`-powered linting. +- Docstrings and examples for `ProtoCollection` and `DelimitedMessageFactory`. + +### Changed +- **Breaking**: rename `ProtoCollection.compile()` to `compiled()`. +- **Breaking**: `compiler_path` is now a parameter of `compiled()` instead of the `ProtoCollection` constructor. +- Replace the use of protobuf `GetMessages` with `descriptor_pool` and `GetMessageClassesForFiles`. +- Remove `tox.ini` and `setup.cfg`; enhance `pyproject.toml`. + +--- + +## [0.2.0] - 2024-02-04 + +### Added +- Add Google address book example to README. +- Add Python 3.12 support. +- Add `protoc` version matrix to CI actions. + +### Fixed +- Pin `protoc` to version `3.20.3`; add a "latest" build. + +### Changed +- Remove `ci` directory; centralize requirements. +- Migrate to `tox` 4 and `tox-gh`. + +--- + +## [0.1.0] - 2023-06-08 + +### Added +- `ProtoCollection.compiler_version()` method added. +- Python 3.11 support added. +- `protoc` pinned to version `23.2` in CI; `arduino/setup-protoc` upgraded from `v1` to `v2`. +- `protoc --version` step added to CI workflows. +- `long_description_content_type` added to `setup.py`. + +### Changed +- Improve robustness of `test_compiler_version()` and `test_compile_invalid_source()`. +- Increase overall test robustness; add dependency tests. + +--- + +## [0.0.18] - 2022-11-17 + +### Changed +- Remove the use of `api_implementation` warning check. +- Replace `bytearray` with `bytes` in `bytes_read()` and `message_read()` return values. +- README converted from RST to Markdown. + +--- + +## [0.0.17] - 2022-04-28 + +### Changed +- `DelimitedMessageFactory` stream type changed from `io.BytesIO` to `typing.BinaryIO`. + +--- + +## [0.0.16] - 2022-04-24 + +### Changed +- **Breaking**: `bytes_read()` and `message_read()` generators now yield `(offset, data)` tuples instead of bare data, prefixing each message with its offset in the input stream. + +--- + +## [0.0.15] - 2022-04-14 + +### Added +- `bytes_read()` generator added to `DelimitedMessageFactory` to yield raw byte chunks. +- `message_read()` generator added to `DelimitedMessageFactory` to yield decoded protobuf messages. + +### Changed +- **Breaking**: rename `DelimitedMessage` to `DelimitedMessageFactory`. +- `DelimitedMessageFactory` now dispatches `read()` to either `bytes_read()` or `message_read()` depending on whether `message_type` is set. + +--- + +## [0.0.14] - 2022-04-14 + +### Added +- `DelimitedMessage` class added (later renamed to `DelimitedMessageFactory`) for reading and writing length-delimited protobuf messages. +- `ProtoCollection` now parses the `FileDescriptorSet` and populates `.descriptor_set` and `.messages` using `GetMessages`. + +--- + +## [0.0.13] - 2022-04-14 + +### Added +- `ProtoModule.compiled()` convenience method added, wrapping single-module compilation via `ProtoCollection`. +- Duplicate proto detection: `add_proto()` now raises `KeyError` if the same `file_path` is added twice. +- Encode and decode tests added. +- `protoc` installation added to CI. + +### Changed +- Rename CI workflow file `github-actions.yml` to `test.yml`. + +--- + +## [0.0.12] - 2022-04-13 + +### Changed +- **Breaking**: `ProtoModule().proto_source` renamed to `.source`. +- **Breaking**: `ProtoCollection().file_descriptor_set` renamed to `.descriptor_data`. +- `importlib.util` used instead of bare `importlib` for module loading. + +--- + +## [0.0.11] - 2022-04-13 + +### Changed +- **Breaking**: rename `ProtoDict` to `ProtoCollection`; `.protos` dict renamed to `.modules`. +- **Breaking**: `ProtoModule` fields renamed: `module_core_name` → `name`, `content` → `proto_source`, `module_source` → `py_source`, `module` → `py`. +- `ProtoModule` now uses `importlib.util.spec_from_loader` and `module_from_spec` for module creation. +- `ProtoCollection` gains `.file_descriptor_set` attribute (serialized `FileDescriptorSet` bytes). + +--- + +## [0.0.10] - 2022-04-12 + +### Fixed +- `raise_for_errs()`: early return when `errs` is empty (no exception raised on empty stderr). + +### Changed +- Linting fixes in `entities.py`. + +--- + +## [0.0.9] - 2022-04-12 + +### Added +- `__init__.py` files added to package directories to allow package importing. +- `raise_for_errs()` and `add_init_files()` extracted as separate methods on `ProtoCollection`. + +### Changed +- `ProtoModule.__init__` now accepts `file_path` as `Path` instead of `str`. +- Allow injection of a `global_scope` dict into module execution. +- Tolerate `protoc` warnings for unused `.proto` files (only raise on real errors). + +--- + +## [0.0.8] - 2022-04-12 + +### Changed +- License changed from LGPL-3.0 to MIT before making the source public. + +--- + +## [0.0.7] - 2022-04-12 + +### Changed +- Version bump only (no functional changes). + +--- + +## [0.0.6] - 2022-04-12 + +### Fixed +- Fix `protoc_command` extension: pass list of string paths instead of dict keys directly. + +--- + +## [0.0.5] - 2022-04-12 + +### Added +- Type annotations added throughout (`Dict`, class-level field declarations). +- Index protos by full `file_path` instead of `module_core_name` to avoid conflicts. +- Use a single `TemporaryDirectory` for both source and output instead of two nested ones. +- `marshal()` static method added to write proto sources to the temp directory. + +--- + +## [0.0.4] - 2022-04-11 + +### Changed +- `ProtoModule.__init__` signature changed: `file_name: str` replaced by `file_path: str`; `package_path` derived from `Path(file_path).parent` instead of being a constructor argument. +- `protoc` invocation refactored to use per-proto paths derived from `file_path`. + +--- + +## [0.0.3] - 2022-04-11 + +### Added +- Core `ProtoModule` and `ProtoDict` classes introduced in `src/proto_topy/entities.py`. +- `ProtoDict.compile()` method: writes proto sources to a temp dir, invokes `protoc`, loads generated Python modules. +- `NoCompiler` and `CompilationFailed` exceptions added. +- Initial test suite added. + +### Changed +- Removed docs/, future/, readthedocs scaffolding inherited from cookiecutter template. +- Simplified CI workflow; removed Sphinx documentation build. +- `setup.py` rewritten to read version and requirements dynamically. + +--- + +## [0.0.2] - 2022-04-10 + +### Added +- Initial project skeleton. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 888a1c7..962fdab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,51 +1,74 @@ ## Linting & Testing - pip install -e '.[dev]' - pre-commit autoupdate +See [TESTING.md](TESTING.md) for the full test strategy and coverage details. + + uv pip install -e '.[dev]' pre-commit install - pre-commit run --all-files - tox + tox -e lint # ruff check + ty check + tox # all Python versions + +`tox` uses the `protoc` compiler found in `PATH`. The CI workflow tests against protoc `25.x` and `34.x`. +On macOS, install protoc with `brew install protobuf`. -`tox` uses by default the installed `protoc` compiler. The gh repo `test` workflow considers `protoc` in various versions (`3.20.3`, `21.12`, `25.x`). -Contributors using a macOS workstation shall have a look at `tests/tox_mac.sh` to test against `protobuf`, `protobuf@21` and `protobuf@3` bottles. - ## Releasing -If dev branch `test` workflow succeeds, a new version can be released. +### 1. Create a release branch + +Check the current version in `pyproject.toml`: + + grep current_version pyproject.toml # e.g. -> 1.0.5 + +Create a branch for the next version (without the `rc` suffix): + + git checkout -b release/2.0.0 + +### 2. Make your changes + +Implement bug fixes, new features, dependency updates, README and CHANGELOG notes, etc. Commit and push. + +### 3. Bump to release candidate + +Push the branch to remote first, then bump: + + git push -u origin release/2.0.0 + bumpver update --major --tag=rc # 1.0.5 -> 2.0.0rc0 + +bumpver commits, tags `v2.0.0rc0`, and pushes automatically. + +### 4. Open a pull request + +Open a PR from `release/2.0.0` against `main`. Ensure: +- the `test` workflow jobs are all green +- Codecov still reports coverage at https://app.codecov.io/gh/decitre/python-proto-topy + +If further changes are needed, iterate: + + bumpver update --tag-num # 2.0.0rc0 -> 2.0.0rc1 + +### 5. Bump to final version + +Once the RC is green: -1. Version + bumpver update --tag=final # 2.0.0rc1 -> 2.0.0 - Use `bumpver` on dev branches. First, with `--dry`, then without :) - - Breaking changes: - - bumpver update --major --dry - - Additional features: - - bumpver update --minor --dry - - Other: - - bumpver update --patch --dry +Confirm the workflow is still green. -2. Merge +### 6. Merge into `main` - To `main` +### 7. Publish on PyPI -3. Publish +Create a release in the [GitHub UI](https://github.com/decitre/python-proto-topy/releases): - Publishing to PyPi is done through the creation of a release in the [Github UI](https://github.com/decitre/python-proto-topy/releases): - - "Draft a new release" - - Choose the tag created in step 1 - - Use it to name the release - - Add the changes to the description field - - "Publish release" - - Check the `release` [action](https://github.com/decitre/python-proto-topy/actions/workflows/release.yml) +- "Draft a new release" +- Choose the tag `v2.0.0` created by `bumpver` in step 5 (not the `rc` tags) +- Use the version as the release name +- Add the changes to the description +- "Publish release" +- Check the `release` [action](https://github.com/decitre/python-proto-topy/actions/workflows/release.yml) -4. Install published package +### 8. Verify the published package - In a dedicated `venv`, do: +In a dedicated virtualenv: - pip install proto_topy== - python -c "import proto_topy as pt; print(pt.__version__, pt.ProtoCollection().compiler_version())" + uv pip install proto-topy==2.0.0 + python -c "import proto_topy as pt; print(pt.__version__, pt.ProtoCollection().compiler_version())" \ No newline at end of file diff --git a/LICENSE b/LICENSE index 21dfbd1..24872c0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -proto-topy - Yet another tool that compiles .proto strings and import the outcome Python modules. +proto-topy - Compile protobuf strings into Python module objects at runtime. Copyright (c) 2022, Emmanuel Decitre. diff --git a/README.md b/README.md index 7629ba4..f103326 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,22 @@ [![test][test_badge]][test_target] +[![codecov][codecov_badge]][codecov] [![version][version_badge]][pypi] [![wheel][wheel_badge]][pypi] [![python version][python_versions_badge]][pypi] [![python implementation][python_implementation_badge]][pypi] -A Python package that -- takes a `str` containing protobuf messages definitions -- returns a `types.ModuleType` instance +Compile protobuf strings into Python module objects at runtime. — no files, no code generation step: + +```python +from proto_topy import ProtoModule + +module = ProtoModule( + source="message Hello { optional string name = 1; }", +).compiled() + +msg = module.py.Hello(name="world") +assert msg.name == "world" +``` It is useful for programs needing to en/decode protobuf messages for which the definition is provided as a string at runtime. @@ -18,25 +28,21 @@ Prerequisite: `proto-topy` needs [protoc][protoc] to be installed. On macOS, a s ## single proto example: address book -Adaptation of the `protocolbuffers` [example](https://github.com/protocolbuffers/protobuf/tree/main/examples): +Adaptation of the `protocolbuffers` [example](https://github.com/protocolbuffers/protobuf/tree/main/examples) from Google tutorial: ```python +from io import BytesIO import requests -from pathlib import Path from proto_topy import ProtoModule -# Retrieve protobuf messages definitions as a string example_source = requests.get( "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/" "examples/addressbook.proto").text -example_path = Path( - "protocolbuffers/protobuf/blob/main/examples/addressbook.proto") - -# Compile and import -module = ProtoModule(file_path=example_path, source=example_source).compiled() +module = ProtoModule(source=example_source).compiled() -# Produce a serialized address book +# Serialize an address book +buffer = BytesIO() address_book = module.py.AddressBook() person = address_book.people.add() person.id = 111 @@ -45,51 +51,58 @@ person.email = "a.name@mail.com" phone_number = person.phones.add() phone_number.number = "+1234567" phone_number.type = module.py.Person.MOBILE -with open("address_book.data", "wb") as o: - o.write(address_book.SerializeToString()) +buffer.write(address_book.SerializeToString()) -# Use a serialized address book +# Deserialize it back +buffer.seek(0) address_book = module.py.AddressBook() -with open("address_book.data", "rb") as i: - address_book.ParseFromString(i.read()) - for person in address_book.people: - print(person.id, person.name, person.email, phone_number.number) +address_book.ParseFromString(buffer.read()) +for person in address_book.people: + assert person.id == 111 + assert person.name == "A Name" + assert person.email == "a.name@mail.com" + ``` ## multiple protos example -When several `.proto` need to be considered, use a `ProtoCollection`: +When several definition strings need to be considered, use a `ProtoCollection`: ```python -import sys -from pathlib import Path from proto_topy import ProtoModule, ProtoCollection +from google.protobuf.timestamp_pb2 import Timestamp module1 = ProtoModule( - file_path=Path("p1/p2/other2.proto"), + file_path="p1/p2/other2.proto", source=""" syntax = "proto3"; import "google/protobuf/timestamp.proto"; message OtherThing2 { google.protobuf.Timestamp created = 1; - };""" + }""" ) module2 = ProtoModule( - file_path=Path("p3/p4/test6.proto"), + file_path="p3/p4/test6.proto", source=""" syntax = "proto3"; import "p1/p2/other2.proto"; message Test6 { OtherThing2 foo = 1; - };""" + }""" ) collection = ProtoCollection(module1, module2).compiled() -sys.modules.update({proto.name: proto.py - for proto in collection.modules.values()}) -print(sys.modules['test6'].Test6, - sys.modules['other2'].OtherThing2) + +Test6 = collection.modules["p3/p4/test6.proto"].py.Test6 +OtherThing2 = collection.modules["p1/p2/other2.proto"].py.OtherThing2 + +ts = Timestamp(seconds=1234567890) +thing = OtherThing2(created=ts) +msg = Test6(foo=thing) + +assert msg.foo.created.seconds == 1234567890 + ``` ## Stream of delimited messages @@ -97,25 +110,21 @@ To decode a stream of contiguous protobuf messages of the same type, use `Delimi ```python from io import BytesIO -from pathlib import Path from proto_topy import ProtoModule, DelimitedMessageFactory -# Generate Python module module = ProtoModule( - file_path=Path("int32_streams.proto"), source=""" syntax = "proto3"; - message TestInt { int32 val = 1; };""" + message TestInt { int32 val = 1; } + """ ).compiled() -# Feed a DelimitedMessageFactory with a sequence of TestInt instances for a range of 10 ints integers = (module.py.TestInt(val=val) for val in range(10)) factory = DelimitedMessageFactory(BytesIO(), *integers) -# Rewind and read the stream of 10 protobuf messages factory.rewind() -for offset_val in factory.message_read(module.py.TestInt): - print(f"TestInt message of val set to {offset_val[1]}") +for i, (offset, msg) in enumerate(factory.message_read(module.py.TestInt)): + assert msg.val == i ``` @@ -127,5 +136,7 @@ for offset_val in factory.message_read(module.py.TestInt): [wheel_badge]: https://img.shields.io/pypi/wheel/proto-topy.svg [python_versions_badge]: https://img.shields.io/pypi/pyversions/proto-topy.svg [python_implementation_badge]: https://img.shields.io/pypi/implementation/proto-topy.svg +[codecov_badge]: https://codecov.io/gh/decitre/python-proto-topy/branch/main/graph/badge.svg +[codecov]: https://codecov.io/gh/decitre/python-proto-topy [tests]: tests/test_proto_topy.py [protoc]: https://protobuf.dev/getting-started/pythontutorial/#compiling-protocol-buffers diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..f2edea2 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,61 @@ +# Testing + +## Test strategy + +Tests are located in [`tests/test_proto_topy.py`](tests/test_proto_topy.py) and cover: + +- **Unit tests** for `ProtoModule`, `ProtoCollection`, and `DelimitedMessageFactory` — including compiler path resolution, proto compilation, message encoding/decoding, and stream handling. +- **Integration tests** that invoke the real `protoc` binary to compile `.proto` sources and exercise the generated Python modules. +- **Error path tests** — invalid proto source, missing dependencies, bad compiler path, type mismatches. + +## Running tests + +### Single Python version + +```bash +tox -e py312 +``` + +### All supported Python versions + +```bash +tox -e clean,py310,py311,py312,py313,py314,report +``` + +The `clean` env erases accumulated coverage data before the run. The `report` env produces terminal, HTML (`htmlcov/`), and XML (`coverage.xml`) reports combining coverage from all versions. + +### Linting + +```bash +tox -e lint +``` + +Runs `ruff check .` and `ty check` (type checking). + +### macOS — multiple protoc versions + +[`tests/tox_mac.sh`](tests/tox_mac.sh) installs and cycles through the Homebrew protobuf bottles (`protobuf@29`, `protobuf@33`, `protobuf`) and runs the full tox suite against each: + +```bash +bash tests/tox_mac.sh +``` + +## Google addressbook example test + +`test_google_addressbook_example` is an end-to-end integration test based on the official Google Protocol Buffers tutorial example: + +> https://github.com/protocolbuffers/protobuf/tree/main/examples + +It fetches [`addressbook.proto`](https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto) — a real-world multi-message schema with nested types (`AddressBook`, `Person`, `PhoneNumber`) — compiles it at runtime via `protoc`, and verifies that a round-trip encode/decode of an address book produces the expected values. + +The HTTP request is recorded and replayed via [vcrpy](https://github.com/kevin1024/vcrpy) / [pytest-recording](https://github.com/kiwicom/pytest-recording), so the test runs offline after the cassette is recorded. The cassette is stored at: + +``` +tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml +``` + +To re-record the cassette (e.g. when the upstream proto changes): + +```bash +pytest tests/test_proto_topy.py::test_google_addressbook_example --record-mode=once +``` diff --git a/pyproject.toml b/pyproject.toml index d8ed7d3..a9830f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,56 +1,72 @@ [build-system] -requires = ["setuptools>=42", "wheel"] +requires = ["setuptools>=82.0.1", "wheel"] build-backend = "setuptools.build_meta" [project] name = "proto-topy" -version = "1.0.5" -description = "Yet another tool that compiles .proto strings and import the outcome Python modules." +description = "Compile protobuf strings into Python module objects at runtime." +license = "MIT" readme = "README.md" -requires-python = ">=3.8" -license = {file = "LICENSE"} +requires-python = ">=3.10" classifiers = [ "Intended Audience :: Developers", - 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3', - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3 :: Only", 'Programming Language :: Python :: Implementation :: CPython', "Operating System :: OS Independent" ] dependencies = ["protobuf>=3.20.3"] +dynamic = ["version"] + +[tool.setuptools.dynamic] +version = {attr = "proto_topy.__version__"} [project.urls] homepage = "https://github.com/decitre/python-proto-topy" [project.optional-dependencies] dev = [ - "pytest", "pytest-recording", "urllib3<2", - "bumpver", "black", "ruff", "pre-commit", - "tox", "build", "twine", "setuptools" + "pytest", + "pytest-recording", + "pytest-cov", + "urllib3", + "bumpver", + "ruff", + "pre-commit", + "tox", + "tox-uv", + "build", + "twine" ] ci = [ - "virtualenv>=16.6.0", "pip>=19.1.1", "setuptools>=18.0.1", - "six>=1.14.0", "tox>=4.12.1", "tox-gh>=1.2" + "tox>=4.12.1", + "tox-gh>=1.2", + "tox-uv", ] -[tool.black] -line-length = 88 -target-version = ['py311'] - [tool.ruff] line-length = 88 exclude = [ - ".tox", ".eggs", "build", "dist", "__pycache__", - "docs/source/conf.py", "old", "env", + ".tox", + ".eggs", + "build", + "dist", + "__pycache__", + "docs/source/conf.py", + "old", + "env", ] lint.ignore = ["COM812", "D103", "E501", "F401", "F403"] lint.select = ["B", "B9", "C", "E", "F", "W", "I001"] +[tool.ruff.lint.isort] +combine-as-imports = true + [tool.ruff.lint.flake8-quotes] inline-quotes = "double" @@ -58,17 +74,95 @@ inline-quotes = "double" max-complexity = 18 [tool.bumpver] -current_version = "1.0.4" -version_pattern = "MAJOR.MINOR.PATCH" -commit = false -tag = false -push = false +current_version = "1.0.5" +version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]" +commit_message = "bump version {old_version} -> {new_version}" +commit = true +tag = true +push = true [tool.bumpver.file_patterns] "pyproject.toml" = [ - 'version = "{version}"', + 'current_version = "{version}"', +] +"src/proto_topy.py" = [ + '__version__ = "{version}"', ] [tool.tox] -# tox 4 supports pyproject.toml only through an inconveninent legacy_tox_ini string... -# So we still keep tox.ini +env_list = [ + "clean", + "lint", + "py310", + "py311", + "py312", + "py313", + "py314", + "report", +] +requires = ["tox-uv"] +ignore_basepython_conflict = true +env.py310.base_python = "{env:TOXPYTHON:python3.10}" +env.py311.base_python = "{env:TOXPYTHON:python3.11}" +env.py312.base_python = "{env:TOXPYTHON:python3.12}" +env.py313.base_python = "{env:TOXPYTHON:python3.13}" +env.py314.base_python = "{env:TOXPYTHON:python3.14}" + +[tool.tox.env_run_base] +set_env.PYTHONPATH = "{toxinidir}/src" +set_env.PYTHONUNBUFFERED = "yes" +pass_env = ["*"] +use_develop = false +deps = [ + "pytest", + "pytest-cov", + "requests", + "pytest-recording", + "urllib3", +] +commands = [ + ["pytest", "--cov=src", "--cov-append", "--cov-report=term-missing", "--cov-report=xml", "-vv", "tests"], +] + +[tool.tox.env.clean] +base_python = "{env:TOXPYTHON:python3}" +skip_install = true +deps = ["coverage"] +commands = [["coverage", "erase"]] + +[tool.tox.env.lint] +base_python = "{env:TOXPYTHON:python3}" +deps = ["ruff", "ty", "pytest", "requests", "pytest-recording"] +commands = [ + ["ruff", "check", "."], + ["ty", "check"], +] + +[tool.tox.env.report] +base_python = "{env:TOXPYTHON:python3}" +skip_install = true +deps = ["coverage"] +commands = [ + ["coverage", "report"], + ["coverage", "html"], + ["coverage", "xml"], +] + +[tool.coverage.run] +omit = ["*/tests/*"] + +[tool.coverage.report] +omit = ["*/tests/*"] + +[tool.tox.env.codecov] +base_python = "{env:TOXPYTHON:python3}" +skip_install = true +deps = ["codecov"] +commands = [["codecov"]] + +[tool.tox.gh.python] +"3.14" = ["py314"] +"3.13" = ["py313"] +"3.12" = ["py312"] +"3.11" = ["py311"] +"3.10" = ["py310"] diff --git a/setup.py b/setup.py index e72b243..f6003f7 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,4 @@ # software. In the absence of such agreement, the use of the software is not # allowed. -from setuptools import setup - -if __name__ == "__main__": - setup() +__import__("setuptools").setup() diff --git a/src/proto_topy.py b/src/proto_topy.py index 7656843..737a5b1 100644 --- a/src/proto_topy.py +++ b/src/proto_topy.py @@ -1,4 +1,5 @@ import importlib.util +import itertools import os import sys import types @@ -8,16 +9,18 @@ from shutil import which from subprocess import PIPE, Popen from tempfile import TemporaryDirectory -from typing import BinaryIO, ClassVar, Dict, Generator, Tuple, Union +from typing import BinaryIO, Dict, Generator, Tuple, Type from google.protobuf import descriptor_pool -from google.protobuf.descriptor_pb2 import FileDescriptorSet +from google.protobuf.descriptor_pb2 import ( + FileDescriptorSet, # type: ignore[attr-defined] +) from google.protobuf.internal.decoder import _DecodeVarint32 from google.protobuf.internal.encoder import _VarintBytes from google.protobuf.message import Message from google.protobuf.message_factory import GetMessageClassesForFiles -__version__ = version("proto-topy") +__version__ = "1.0.5" logger = getLogger(Path(__file__).name) @@ -49,25 +52,30 @@ class ProtoModule: package_path: Path file_path: Path source: str - py_source: str - py: types.ModuleType + py_source: str | None + py: types.ModuleType | None - def __init__(self, file_path: Path, source: str): - self.file_path = file_path + _counter = itertools.count(1) + + def __init__(self, source: str, file_path: Path | str | None = None): + if file_path is None: + file_path = f"definition_{next(self._counter)}.proto" + self.file_path = Path(file_path) self.name, _, _ = self.file_path.name.partition(".proto") self.source = source self.package_path = self.file_path.parent - self.py = None - self.py_source = None + self.py: types.ModuleType | None = None + self.py_source: str | None = None - def _set_module(self, content: str, global_scope: dict = None): + def _set_module(self, content: str, global_scope: dict | None = None): self.py_source = content spec = importlib.util.spec_from_loader(self.name, loader=None) + assert spec is not None compiled_content = compile(content, self.name, "exec") self.py = importlib.util.module_from_spec(spec) exec(compiled_content, self.py.__dict__) - def compiled(self, compiler_path: Path = None) -> "ProtoModule": + def compiled(self, compiler_path: Path | None = None) -> "ProtoModule": """ Returns the ProtoModule instance in a compiled state: @@ -101,15 +109,15 @@ class ProtoCollection: - messages: A dictionary of protobuf messages classes indexed by their proto names """ - modules: Dict[Path, ProtoModule] - descriptor_data: bytes - descriptor_set: FileDescriptorSet + modules: Dict[Path | str, ProtoModule] + descriptor_data: bytes | None + descriptor_set: "FileDescriptorSet | None" messages: dict def __init__(self, *protos: ProtoModule): self.modules = {} - self.descriptor_data = None - self.descriptor_set = None + self.descriptor_data: bytes | None = None + self.descriptor_set: "FileDescriptorSet | None" = None self.messages = {} for proto in protos or []: @@ -119,19 +127,20 @@ def add_proto(self, proto: ProtoModule): if proto.file_path in self.modules: raise KeyError(f"{proto.file_path} already added") self.modules[proto.file_path] = proto + self.modules[str(proto.file_path)] = proto @staticmethod def _get_compiler_path() -> Path: if "PROTOC" in os.environ and os.path.exists(os.environ["PROTOC"]): compiler_path = Path(os.environ["PROTOC"]) else: - compiler_path = Path(which("protoc")) + compiler_path = Path(which("protoc") or "") if not compiler_path.is_file(): raise FileNotFoundError("protoc compiler not found") return compiler_path def compiled( - self, compiler_path: Path = None, global_scope: dict = None + self, compiler_path: Path | None = None, global_scope: dict | None = None ) -> "ProtoCollection": if not compiler_path: compiler_path = ProtoCollection._get_compiler_path() @@ -169,14 +178,14 @@ def compiled( self.descriptor_data = f.read() self.descriptor_set = FileDescriptorSet.FromString(self.descriptor_data) - pool = descriptor_pool.DescriptorPool() + pool = descriptor_pool.DescriptorPool() # type: ignore[possibly-missing-implicit-call] for file_descriptor_proto in self.descriptor_set.file: pool.Add(file_descriptor_proto) self.messages = GetMessageClassesForFiles( [fdp.name for fdp in self.descriptor_set.file], pool ) - self._add_init_files(dir) + self._add_init_files(Path(dir)) sys.path.append(dir) for proto in self.modules.values(): @@ -187,7 +196,7 @@ def compiled( sys.path.pop() return self - def compiler_version(self, compiler_path: Path = None) -> str: + def compiler_version(self, compiler_path: Path | None = None) -> str | None: """ Returns the result of a `protoc --version` command. @@ -202,8 +211,9 @@ def compiler_version(self, compiler_path: Path = None) -> str: [], raise_exception=True, ) - if outs: + if outs: # pragma: no branch return outs.split()[-1].decode() + return None # pragma: no cover @staticmethod def _do_compile( @@ -217,7 +227,6 @@ def _do_compile( compile_command.extend(compile_to_py_options) compile_command.extend(proto_source_files) compilation = Popen(compile_command, stdout=PIPE, stderr=PIPE) - compilation.wait() outs, errs = compilation.communicate() if raise_exception: ProtoCollection._raise_for_errs(errs) @@ -260,24 +269,21 @@ class DelimitedMessageFactory: """ def __init__( - self, stream: BinaryIO, *messages: Message, message_type: ClassVar = None + self, stream: BinaryIO, *messages: Message, message_type: Type[Message] | None = None ): self.stream = stream self.message_type = message_type self.offset = 0 if message_type is None: - self.read = self.bytes_read + self.read = self.bytes_read # type: ignore[method-assign] else: - self.read = self.message_read - if messages: + self.read = self.message_read # type: ignore[method-assign] + if messages: # pragma: no branch self.write(*messages) def read( self, - ) -> Union[ - Generator[Tuple[int, Message], None, None], - Generator[Tuple[int, bytearray], None, None], - ]: + ) -> Generator[Tuple[int, Message], None, None] | Generator[Tuple[int, bytearray], None, None]: raise NotImplementedError() def write(self, *messages: Message): @@ -308,7 +314,7 @@ def bytes_read(self) -> Generator[Tuple[int, bytes], None, None]: :return: tuple of message offset and message bytes """ buf = bytearray(self.stream.read(10)) - while buf: + while buf: # pragma: no branch msg_len, new_pos = _DecodeVarint32(buf, 0) self.offset += new_pos buf = buf[new_pos:] @@ -325,7 +331,7 @@ def bytes_read(self) -> Generator[Tuple[int, bytes], None, None]: buf.extend(self.stream.read(10 - len(buf))) def message_read( - self, message_type: ClassVar = None + self, message_type: Type[Message] | None = None ) -> Generator[Tuple[int, Message], None, None]: """ :return: tuple of message offset and decoded bytes @@ -333,7 +339,8 @@ def message_read( buf = bytearray(self.stream.read(10)) self.offset += 10 message_type = message_type or self.message_type - while buf: + assert message_type is not None + while buf: # pragma: no branch message = message_type() msg_len, new_pos = _DecodeVarint32(buf, 0) self.offset += new_pos diff --git a/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml b/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml index d69dacb..0ac364a 100644 --- a/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml +++ b/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml @@ -9,24 +9,28 @@ interactions: Connection: - keep-alive User-Agent: - - python-requests/2.31.0 + - python-requests/2.32.5 method: GET uri: https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto response: body: - string: !!binary | - H4sIAAAAAAAAA22UXU/bMBSG7/Mrjno9ko1uN0RcgKg2tEEr6LSLCVVucpqYJrFnHzPQxH/fsZ2k - 6YZURY3j877P+bCzDO4R4W5xcXWzSNsSdsqA7PjZCpKqA9GVsHWyKXnVknGFX7VpkmX8g1tFeAb3 - 64u7ddi5uL0CEpUFYRCcRR8FhWpb7MgCKShxJzsEi1Fn2OO1yJEyUjQ2BVjX+BI0OkWghSFQO6Ca - A186Es+BcmUUqUI1cOl2OzQj01pBhcQ4rHtSoqYafotmT7VRrqqjkLSwkw0GZi9rsBHEJPgsWt2g - fceEeObVaiJtz7KsxCdslPY+lVJVgymnleme4WQbGbJSFTYbM0m8ws9YnhKLRphQ1IekT+McZkFh - PssTLYq9qHAsQ54kstWKU59Fw2jGRhnJFi0xaRqWONjb+NofmUzMH8WT2Bx9VDq0N3xoXUOS0974 - mlim4j5jfrRloGNizjvt65QOsBHEzo6DlCM0Gza1thNtCL4oS4PWXiq1Xw0hA/z/kJMMClsLo9/M - of/kLSxzBp/PsUervmTpom9sOgGYWL+lPjGv1JvGvDypSyWpdtujsRimYuzcMF9ZpcYp0dsJyL9O - EwjOzrKVfUj6f7Bibcb4kwDw0ZRdBX2ZP+S8JDuan4Is+f00B2Ch75385RCur6Bz7RZNOEbhMOig - lB6EsBWy4cg5jyEA8n5Y1arD9YvGYAhws7y8/rbgPe/z8P5lebMYvAF+LO++Bmd+e/UaI7SXuY3+ - UWhgj2ujwsGP/GOqZVBjOK9TMe3/++n9GJj7UzqUPl0PhwZ4HmnjdBkUzuFTnryGOi+dARHHA7Y8 - H/GK4Oo8OssXEF9b8RKymI4tmIxTyOZAFnujUXG/Y1KvY5cPrfwLfa1h8YAFAAA= + string: "// See README.md for information and build instructions.\n//\n// Note: + START and END tags are used in comments to define sections used in\n// tutorials. + \ They are not part of the syntax for Protocol Buffers.\n//\n// To get an + in-depth walkthrough of this file and the related examples, see:\n// https://developers.google.com/protocol-buffers/docs/tutorials\n\n// + [START declaration]\nsyntax = \"proto3\";\npackage tutorial;\n\nimport \"google/protobuf/timestamp.proto\";\n// + [END declaration]\n\n// [START java_declaration]\noption java_multiple_files + = true;\noption java_package = \"com.example.tutorial.protos\";\noption java_outer_classname + = \"AddressBookProtos\";\n// [END java_declaration]\n\n// [START csharp_declaration]\noption + csharp_namespace = \"Google.Protobuf.Examples.AddressBook\";\n// [END csharp_declaration]\n\n// + [START go_declaration]\noption go_package = \"github.com/protocolbuffers/protobuf/examples/go/tutorialpb\";\n// + [END go_declaration]\n\n// [START messages]\nmessage Person {\n string name + = 1;\n int32 id = 2; // Unique ID number for this person.\n string email + = 3;\n\n enum PhoneType {\n MOBILE = 0;\n HOME = 1;\n WORK = 2;\n + \ }\n\n message PhoneNumber {\n string number = 1;\n PhoneType type + = 2;\n }\n\n repeated PhoneNumber phones = 4;\n\n google.protobuf.Timestamp + last_updated = 5;\n}\n\n// Our address book file is just one of these.\nmessage + AddressBook {\n repeated Person people = 1;\n}\n// [END messages]\n" headers: Accept-Ranges: - bytes @@ -47,35 +51,35 @@ interactions: Cross-Origin-Resource-Policy: - cross-origin Date: - - Sat, 03 Feb 2024 14:11:23 GMT + - Sun, 15 Mar 2026 09:40:15 GMT ETag: - W/"3609494263025aad9f621723990edef51c0d69d5dbaa781f6a5867e4e66b4c94" Expires: - - Sat, 03 Feb 2024 14:16:23 GMT + - Sun, 15 Mar 2026 09:45:15 GMT Source-Age: - '0' Strict-Transport-Security: - max-age=31536000 Vary: - - Authorization,Accept-Encoding,Origin + - Authorization,Accept-Encoding Via: - 1.1 varnish X-Cache: - - HIT + - MISS X-Cache-Hits: - - '1' + - '0' X-Content-Type-Options: - nosniff X-Fastly-Request-ID: - - ed62ed6bb3831cc99bc14b1b39f32cd2859affe4 + - f37c88d272af5f602d452ea7c7722efc78bc3c39 X-Frame-Options: - deny X-GitHub-Request-Id: - - 4716:250E4C:2B13C31:2CCC87B:65BE4834 + - 2BE8:341CE5:17E3062:1B39CC2:69B67E7E X-Served-By: - - cache-fra-eddf8230121-FRA + - cache-fra-eddf8230030-FRA X-Timer: - - S1706969484.625163,VS0,VE134 + - S1773567615.995590,VS0,VE158 X-XSS-Protection: - 1; mode=block status: diff --git a/tests/test_proto_topy.py b/tests/test_proto_topy.py index 4e0bf1d..4630aac 100644 --- a/tests/test_proto_topy.py +++ b/tests/test_proto_topy.py @@ -15,7 +15,7 @@ ProtoModule, ) -protoc_path = Path(which("protoc") or os.environ.get("PROTOC")) +protoc_path = Path(which("protoc") or os.environ.get("PROTOC") or "") @pytest.fixture() @@ -37,6 +37,12 @@ def unlink_proto_file(path_str: str) -> Path: return proto_path +def test_proto_module_auto_generated_file_path(): + proto = ProtoModule(source="") + assert proto.file_path.name.startswith("definition_") + assert proto.file_path.suffix == ".proto" + + def test_add_proto(): test1_proto = unlink_proto_file("test1.proto") proto = ProtoModule(file_path=test1_proto, source="") @@ -89,7 +95,7 @@ def test_compile_redundant_proto(): def test_compile_minimal_proto(): - from google.protobuf.timestamp_pb2 import Timestamp + from google.protobuf.timestamp_pb2 import Timestamp # type: ignore[attr-defined] test5_proto = unlink_proto_file("test5.proto") proto = ProtoModule( @@ -102,6 +108,7 @@ def test_compile_minimal_proto(): } """, ).compiled(protoc_path) + assert proto.py is not None sys.modules["test5"] = proto.py atest5 = proto.py.Test5() assert isinstance(atest5.created, Timestamp) @@ -110,7 +117,7 @@ def test_compile_minimal_proto(): def test_compile_minimal_proto_in_a_package(): - from google.protobuf.timestamp_pb2 import Timestamp + from google.protobuf.timestamp_pb2 import Timestamp # type: ignore[attr-defined] thing_proto = unlink_proto_file("p1/p2/p3/thing.proto") proto = ProtoModule( @@ -123,6 +130,8 @@ def test_compile_minimal_proto_in_a_package(): } """, ).compiled(protoc_path) + assert proto.py is not None + assert proto.py_source is not None assert "# source: p1/p2/p3/thing.proto" in proto.py_source.split("\n") sys.modules["thing"] = proto.py athing = proto.py.Thing() @@ -171,7 +180,7 @@ def test_compile_ununsed_dependency(): def test_compile_simple_dependency(): - from google.protobuf.timestamp_pb2 import Timestamp + from google.protobuf.timestamp_pb2 import Timestamp # type: ignore[attr-defined] test_proto = unlink_proto_file("p3/p4/test6.proto") proto_module = ProtoModule( @@ -198,10 +207,12 @@ def test_compile_simple_dependency(): ) modules = ProtoCollection(proto_module, other_proto_module) modules.compiled(compiler_path=protoc_path) - sys.modules.update({proto.name: proto.py for proto in modules.modules.values()}) - atest6 = modules.modules[test_proto].py.Test6() + sys.modules.update({proto.name: proto.py for proto in modules.modules.values()}) # type: ignore[arg-type] + m = modules.modules[test_proto] + assert m.py is not None + atest6 = m.py.Test6() assert isinstance(atest6.foo.created, Timestamp) - for proto_module in modules.modules.values(): + for proto_module in set(modules.modules.values()): del sys.modules[proto_module.name] unlink_proto_file("p3/p4/test6.proto") unlink_proto_file("p1/p2/other2.proto") @@ -216,6 +227,8 @@ def test_encode_message(): proto2 = ProtoModule(file_path=test8_proto, source=proto_source.format(n=8)) ProtoCollection(proto1, proto2).compiled(compiler_path=protoc_path) + assert proto1.py is not None + assert proto2.py is not None assert array("B", proto1.py.Test7(foo=124).SerializeToString()) == array( "B", [8, 124] ) @@ -232,6 +245,7 @@ def test_decode_message(): file_path=test9_proto, source='syntax = "proto3"; message Test9 { int32 foo = 1; };', ).compiled(protoc_path) + assert proto.py is not None aTest9 = proto.py.Test9() aTest9.ParseFromString(bytes(array("B", [8, 124]))) assert aTest9.foo == 124 @@ -244,11 +258,12 @@ def test_decode_messages_stream(): file_path=test10_proto, source='syntax = "proto3"; message Test10 { int32 foo = 1; };', ).compiled(protoc_path) + assert proto.py is not None factory = DelimitedMessageFactory( BytesIO(), *(proto.py.Test10(foo=foo) for foo in [1, 12]) ) factory.stream.seek(0) - assert [thing.foo for _, thing in factory.message_read(proto.py.Test10)] == [1, 12] + assert [thing.foo for _, thing in factory.message_read(proto.py.Test10)] == [1, 12] # type: ignore[union-attr] unlink_proto_file("test10.proto") @@ -258,6 +273,7 @@ def test_decode_messages_stream2(): file_path=test11_proto, source='syntax = "proto3"; message Test11 { int32 foo = 1; };', ).compiled(protoc_path) + assert proto.py is not None message = DelimitedMessageFactory( BytesIO(), *(proto.py.Test11(foo=foo) for foo in [1, 12]) ) @@ -265,7 +281,8 @@ def test_decode_messages_stream2(): for fn in message.read, message.bytes_read: message.stream.seek(0) foos = [] - for offset_data in fn(): + for offset_data in fn(): # type: ignore[call-arg] + assert proto.py is not None aTest11 = proto.py.Test11() aTest11.ParseFromString(offset_data[1]) foos.append(aTest11.foo) @@ -273,6 +290,95 @@ def test_decode_messages_stream2(): unlink_proto_file("test11.proto") +def test_get_compiler_path_from_env(monkeypatch, tmp_path): + fake_protoc = tmp_path / "protoc" + fake_protoc.touch() + monkeypatch.setenv("PROTOC", str(fake_protoc)) + path = ProtoCollection._get_compiler_path() + assert path == fake_protoc + + +def test_get_compiler_path_not_found(monkeypatch, tmp_path): + # PROTOC points to a non-existent file, and protoc is not on PATH + monkeypatch.delenv("PROTOC", raising=False) + monkeypatch.setenv("PATH", str(tmp_path)) # empty dir — no protoc here + with pytest.raises((FileNotFoundError, TypeError)): + ProtoCollection._get_compiler_path() + + +def test_compiled_auto_detects_compiler(): + test_proto = unlink_proto_file("test_auto.proto") + proto = ProtoModule( + file_path=test_proto, + source='syntax = "proto3"; message TestAuto { int32 x = 1; };', + ).compiled() # no compiler_path — auto-detect via PATH + assert proto.py is not None + assert proto.py.TestAuto(x=7).x == 7 + unlink_proto_file("test_auto.proto") + + +def test_compiler_version_auto_detects(): + version = ProtoCollection().compiler_version() # no compiler_path + assert version is not None and tuple(map(int, version.split("."))) > (3, 0, 0) + + +def test_delimited_message_factory_with_message_type(): + test_proto = unlink_proto_file("test_dmf.proto") + proto = ProtoModule( + file_path=test_proto, + source='syntax = "proto3"; message TestDmf { int32 v = 1; };', + ).compiled(protoc_path) + assert proto.py is not None + msg_type = proto.py.TestDmf + factory = DelimitedMessageFactory( + BytesIO(), msg_type(v=5), msg_type(v=9), message_type=msg_type + ) + factory.stream.seek(0) + results = [msg.v for _, msg in factory.read()] # type: ignore[union-attr,call-arg] + assert results == [5, 9] + unlink_proto_file("test_dmf.proto") + + +def test_delimited_message_factory_read_raises(): + factory = DelimitedMessageFactory.__new__(DelimitedMessageFactory) + with pytest.raises(NotImplementedError): + list(DelimitedMessageFactory.read(factory)) + + +def test_delimited_message_factory_write_type_error(): + test_proto_a = unlink_proto_file("test_dmf_a.proto") + test_proto_b = unlink_proto_file("test_dmf_b.proto") + proto_a = ProtoModule( + file_path=test_proto_a, + source='syntax = "proto3"; message DmfA { int32 v = 1; };', + ).compiled(protoc_path) + proto_b = ProtoModule( + file_path=test_proto_b, + source='syntax = "proto3"; message DmfB { int32 v = 1; };', + ).compiled(protoc_path) + assert proto_a.py is not None + assert proto_b.py is not None + factory = DelimitedMessageFactory(BytesIO(), proto_a.py.DmfA(v=1)) + with pytest.raises(TypeError): + factory.write(proto_b.py.DmfB(v=2)) + unlink_proto_file("test_dmf_a.proto") + unlink_proto_file("test_dmf_b.proto") + + +def test_delimited_message_factory_rewind(): + test_proto = unlink_proto_file("test_rewind.proto") + proto = ProtoModule( + file_path=test_proto, + source='syntax = "proto3"; message TestRewind { int32 v = 1; };', + ).compiled(protoc_path) + assert proto.py is not None + factory = DelimitedMessageFactory(BytesIO(), proto.py.TestRewind(v=42)) + factory.rewind() + assert factory.stream.tell() == 0 + assert factory.offset == 0 + unlink_proto_file("test_rewind.proto") + + @pytest.mark.vcr def test_google_addressbook_example(address_book): @@ -283,6 +389,7 @@ def test_google_addressbook_example(address_book): file_path=adressbook_proto, source=address_book, ).compiled(protoc_path) + assert proto.py is not None sys.modules["addressbook"] = proto.py # Produce serialized address book diff --git a/tests/tox_mac.sh b/tests/tox_mac.sh index f68307e..0effe9f 100755 --- a/tests/tox_mac.sh +++ b/tests/tox_mac.sh @@ -1,15 +1,16 @@ -PROTOC_BOTTLES="protobuf@3 protobuf@21 protobuf" +PROTOC_BOTTLES="protobuf@29 protobuf@33 protobuf" YELLOW='\033[33m' NC='\033[0m' brew install $PROTOC_BOTTLES brew unlink $PROTOC_BOTTLES PATHBAK=$PATH +BREW_PREFIX=$(brew --prefix) for bottle in $PROTOC_BOTTLES; do printf "${YELLOW}Use $bottle${NC}\n" brew link --overwrite --force $bottle - PATH="/usr/local/opt/$bottle/bin:$PATHBAK" + PATH="$BREW_PREFIX/opt/$bottle/bin:$PATHBAK" protoc --version tox [[ $? -ne 0 ]] && break diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 1679872..0000000 --- a/tox.ini +++ /dev/null @@ -1,64 +0,0 @@ -[tox] -envlist = - clean, - lint, - {py38,py39,py310,py311,py312}, - report -ignore_basepython_conflict = true - -[gh] -python = - 3.12 = py312 - 3.11 = py311 - 3.10 = py310 - 3.9 = py39 - 3.8 = py38 - -[testenv] -basepython = - 3.8: {env:TOXPYTHON:python3.8} - 3.9: {env:TOXPYTHON:python3.9} - 3.10: {env:TOXPYTHON:python3.10} - 3.11: {env:TOXPYTHON:python3.11} - 3.12: {env:TOXPYTHON:python3.12} - {clean,lint,report,codecov}: {env:TOXPYTHON:python3} -setenv = - PYTHONPATH={toxinidir}/src - PYTHONUNBUFFERED=yes -passenv = - * -usedevelop = false -deps = - pytest - pytest-cov - requests - pytest-recording - urllib3<2 -commands = - {posargs:pytest --cov --cov-report=term-missing -vv tests} - -[testenv:codecov] -deps = - codecov -skip_install = true -commands = - codecov [] - -[testenv:lint] -deps = pre-commit -commands = pre-commit run --all-files - -[testenv:report] -deps = - coverage -skip_install = true -commands = - coverage report - coverage html - coverage xml - -[testenv:clean] -commands = coverage erase -skip_install = true -deps = - coverage From bffb91f9fba3bb6c686fcaa22ecdf3d0191e5b11 Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 20:59:48 +0100 Subject: [PATCH 2/8] bump version 1.0.5 -> 2.0.0rc0 --- pyproject.toml | 2 +- src/proto_topy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9830f1..2722c6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ inline-quotes = "double" max-complexity = 18 [tool.bumpver] -current_version = "1.0.5" +current_version = "2.0.0rc0" version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/proto_topy.py b/src/proto_topy.py index 737a5b1..725d65c 100644 --- a/src/proto_topy.py +++ b/src/proto_topy.py @@ -20,7 +20,7 @@ from google.protobuf.message import Message from google.protobuf.message_factory import GetMessageClassesForFiles -__version__ = "1.0.5" +__version__ = "2.0.0rc0" logger = getLogger(Path(__file__).name) From 125fb1f89d6d53b1495afdbd645b3525a4149b45 Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 21:05:22 +0100 Subject: [PATCH 3/8] bump version 2.0.0rc0 -> 2.0.0rc1 --- pyproject.toml | 2 +- src/proto_topy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2722c6c..fc64d86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ inline-quotes = "double" max-complexity = 18 [tool.bumpver] -current_version = "2.0.0rc0" +current_version = "2.0.0rc1" version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/proto_topy.py b/src/proto_topy.py index 725d65c..43be4b9 100644 --- a/src/proto_topy.py +++ b/src/proto_topy.py @@ -20,7 +20,7 @@ from google.protobuf.message import Message from google.protobuf.message_factory import GetMessageClassesForFiles -__version__ = "2.0.0rc0" +__version__ = "2.0.0rc1" logger = getLogger(Path(__file__).name) From c46f8d2bfe69f09ea0a185a9b7ce9a5430870284 Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 21:10:39 +0100 Subject: [PATCH 4/8] Fix an issue in test.yml --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d556e59..df76b45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,6 @@ jobs: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v7 - run: uv run --extra dev ruff check . - - run: uv run --extra dev ty check test: strategy: From b66453763cfa493f53d9f365d0fcaef506e5d43e Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 21:22:23 +0100 Subject: [PATCH 5/8] iterate to get coverage uploaded --- .github/workflows/test.yml | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df76b45..bd1f559 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,7 @@ jobs: version: ${{ matrix.protoc }} repo-token: ${{ secrets.GITHUB_TOKEN }} - run: uv run --extra ci tox -e py310,py311,py312,py313,py314 + - run: ls -la coverage.xml || echo "coverage.xml not found" - uses: codecov/codecov-action@v5 if: matrix.os == 'ubuntu-latest' && matrix.protoc == '25.x' with: diff --git a/pyproject.toml b/pyproject.toml index fc64d86..f5883ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ deps = [ "urllib3", ] commands = [ - ["pytest", "--cov=src", "--cov-append", "--cov-report=term-missing", "--cov-report=xml", "-vv", "tests"], + ["pytest", "--cov=src", "--cov-append", "--cov-report=term-missing", "--cov-report=xml:{toxinidir}/coverage.xml", "-vv", "tests"], ] [tool.tox.env.clean] From 7f6140296a364742aaaf930b8b6153bbf8ea6a65 Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 21:30:31 +0100 Subject: [PATCH 6/8] iterate to get coverage uploaded --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd1f559..52ada50 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,8 +36,8 @@ jobs: version: ${{ matrix.protoc }} repo-token: ${{ secrets.GITHUB_TOKEN }} - run: uv run --extra ci tox -e py310,py311,py312,py313,py314 - - run: ls -la coverage.xml || echo "coverage.xml not found" - uses: codecov/codecov-action@v5 if: matrix.os == 'ubuntu-latest' && matrix.protoc == '25.x' with: - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true From 25430b6ff986b0106e5b1e6254b9c01727433cbe Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 21:33:51 +0100 Subject: [PATCH 7/8] bump version 2.0.0rc1 -> 2.0.0rc2 --- pyproject.toml | 2 +- src/proto_topy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f5883ce..bcecdad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ inline-quotes = "double" max-complexity = 18 [tool.bumpver] -current_version = "2.0.0rc1" +current_version = "2.0.0rc2" version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/proto_topy.py b/src/proto_topy.py index 43be4b9..ddf47ae 100644 --- a/src/proto_topy.py +++ b/src/proto_topy.py @@ -20,7 +20,7 @@ from google.protobuf.message import Message from google.protobuf.message_factory import GetMessageClassesForFiles -__version__ = "2.0.0rc1" +__version__ = "2.0.0rc2" logger = getLogger(Path(__file__).name) From 9676e6f7c3345499fad7f417ff822dc70400216f Mon Sep 17 00:00:00 2001 From: decitre Date: Mon, 16 Mar 2026 21:36:06 +0100 Subject: [PATCH 8/8] bump version 2.0.0rc2 -> 2.0.0 --- pyproject.toml | 2 +- src/proto_topy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bcecdad..a2bcdfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ inline-quotes = "double" max-complexity = 18 [tool.bumpver] -current_version = "2.0.0rc2" +current_version = "2.0.0" version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/proto_topy.py b/src/proto_topy.py index ddf47ae..88d2e48 100644 --- a/src/proto_topy.py +++ b/src/proto_topy.py @@ -20,7 +20,7 @@ from google.protobuf.message import Message from google.protobuf.message_factory import GetMessageClassesForFiles -__version__ = "2.0.0rc2" +__version__ = "2.0.0" logger = getLogger(Path(__file__).name)