From 0837d6a4afb29b40a62cd09f9c06ab2a0cb05339 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:20:16 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m1/ffi-export/python-types.md Quest-Claim: df0f76b48a213d68 From a424239823ea3e2bd816241dd205366caa68ca8e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:50:05 -0800 Subject: [PATCH 2/6] fix(ffi): generated Python passes mypy and pyright, strict The Python declares the extension it loads as a typed namespace under TYPE_CHECKING, enum variants as type aliases on their enum, a plain enum's constants as nonmembers, and module constants as annotated names; its runtime is annotated, any buffer is `bytes | bytearray | memoryview`, and a socket address is the socket module's tuple. `api_py_types` and `just ffi moq` check both. Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 11 +- ffi/justfile | 5 +- ffi/runtime/native.py | 31 +++++ ffi/runtime/rt.py | 32 ++--- ffi/src/python.rs | 205 ++++++++++++++++++++-------- ffi/tests/fixtures.rs | 16 +++ ffi/tests/pyright.json | 4 + flake.nix | 6 +- quest/m1/ffi-export/README.md | 1 - quest/m1/ffi-export/python-types.md | 16 --- 10 files changed, 232 insertions(+), 95 deletions(-) create mode 100644 ffi/runtime/native.py create mode 100644 ffi/tests/pyright.json delete mode 100644 quest/m1/ffi-export/python-types.md diff --git a/ffi/README.md b/ffi/README.md index c765b8cfe..a01ffee1d 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -318,7 +318,8 @@ Each function, method, and property but an `async def` is the extension's own, b the module or set on its class as it loads, so a call enters it with no Python frame between; the Python declares them under `typing.TYPE_CHECKING`, for type checkers. They take arguments by position or keyword, raise as a `def` would, and carry their -docstrings and signatures. +docstrings and signatures. The module passes mypy and pyright, both strict, so a typed +caller imports it with no ignores. - **Names** are in the module: types in PascalCase without the prefix, members in snake_case without their type's name (`moq.TrackProducer.append_group`). A keyword, a @@ -335,7 +336,7 @@ docstrings and signatures. `Event.Moved(p)`. `Display` is `str()`, `Debug` `repr()`, and `Default` a static `default()`. Constants are module or class attributes, made anew on each read unless a literal. -- **Bytes** passed in are any buffer: `bytes` is read in place, and others, which Python +- **Bytes** passed in are any buffer, `bytes | bytearray | memoryview`: `bytes` is read in place, and others, which Python may change while Rust reads, are copied. Bytes Rust returns, or a record holds, are a read-only `memoryview` lending Rust's buffer until it and its views are released. Those Rust lends a closure or trait method are copied into `bytes`. Strings are `str`, @@ -570,8 +571,8 @@ names and signatures, so [`src/c.rs`](src/c.rs), [`src/cpp.rs`](src/cpp.rs), agree; the runtime the glue includes is [`runtime/rt.rs`](runtime/rt.rs), the one the C++ header includes [`runtime/rt.hpp`](runtime/rt.hpp), the ones Kotlin and its JNI glue include [`runtime/rt.kt`](runtime/rt.kt) and [`runtime/jni.rs`](runtime/jni.rs), the ones -Python and its extension include [`runtime/rt.py`](runtime/rt.py) and -[`runtime/python.rs`](runtime/python.rs), the one the Swift includes +Python and its extension include [`runtime/rt.py`](runtime/rt.py), +[`runtime/native.py`](runtime/native.py), and [`runtime/python.rs`](runtime/python.rs), the one the Swift includes [`runtime/rt.swift`](runtime/rt.swift), the one the Go includes [`runtime/rt.go`](runtime/rt.go), and the ones the Dart and its glue include [`runtime/rt.dart`](runtime/rt.dart) and [`runtime/dart.rs`](runtime/dart.rs). @@ -615,6 +616,8 @@ x86_64 emulator at API 24, the oldest moq's Android library supports, and at API warnings as errors, checks its futures the same way, what frees an object or bytes nobody closed, and the values Python's types bound, and breaks each runtime check, which must raise or abort with its message; the Python with another contract must refuse the library. +`api_py_types` checks the Python with mypy and pyright, both strict, at CPython 3.11, as +`just ffi moq` checks moq's. `tests/api.swift` runs the script through the Swift, built as a module in Swift 6's language mode and type-checked in Swift 5's, checks its futures the same way and that releasing frees, and breaks each runtime check, which must trap or abort with its message. diff --git a/ffi/justfile b/ffi/justfile index bd5c7936b..0e8d987c7 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -96,8 +96,11 @@ moq checkout="": (moq-cpp checkout) rm -rf target/moq-kt kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/moq.kt -d target/moq-kt kotlin -Djava.library.path="$out/target/release" -cp "target/moq-kt:$KOTLINX_COROUTINES" MoqKt - # The Python, through the CPython extension the python feature builds. + # The Python, through the CPython extension the python feature builds, and its types + # through mypy and pyright, as `api_py_types` checks the api fixture's. PYTHONPATH="$out/python" python3 -P -X dev -W error moq/moq.py + mypy --strict --python-version 3.11 --cache-dir target/mypy "$out/python/moq.py" + pyright -p tests/pyright.json "$out/python/moq.py" if [ "{{ os() }}-{{ arch() }}" = linux-aarch64 ]; then echo "skipping moq/moq.swift: nixpkgs builds no Swift for aarch64-linux (quest/m2/uniffi-swift-arm.md)" >&2 else diff --git a/ffi/runtime/native.py b/ffi/runtime/native.py new file mode 100644 index 000000000..dd2ef932e --- /dev/null +++ b/ffi/runtime/native.py @@ -0,0 +1,31 @@ +# The extension `_load` loads, as type checkers see it: the runtime's types, and after +# them each glue function the Python calls by name. Every generated Python module +# declares it under `_typing.TYPE_CHECKING`, in place of the module. + + +class _n: + PENDING: object + + class Object: + __slots__ = () + + class Future: + __slots__ = () + + def poll(self, queue: _n.Queue, /) -> object: ... + + def claim(self) -> int: ... + + def close(self) -> None: ... + + class Queue: + def __init__(self, loop: _asyncio.AbstractEventLoop, /) -> None: ... + + def fileno(self) -> int: ... + + def take(self, read: bool, /) -> list[int]: ... + + class Node: + def __init__(self, queue: _n.Queue, /) -> None: ... + + def token(self) -> int: ... diff --git a/ffi/runtime/rt.py b/ffi/runtime/rt.py index ceb63ac62..11f68d87e 100644 --- a/ffi/runtime/rt.py +++ b/ffi/runtime/rt.py @@ -3,7 +3,7 @@ # `_` but `Waker`, so no exported name clashes with them. -class _Future(_n.Future): +class _Future(_n.Future, _typing.Generic[_T]): """A Rust future its first poll left pending: awaiting it polls it on the running event loop until it finishes. Cancelling the awaiting task frees it, which cancels it in Rust and loses nothing, as does dropping it unawaited. Like a coroutine, it is awaited @@ -11,7 +11,7 @@ class _Future(_n.Future): __slots__ = () - def __await__(self): + def __await__(self) -> _typing.Generator[_typing.Any, None, _T]: loop = _get_running_loop() w = _wakeup(loop) token = self.claim() @@ -19,7 +19,7 @@ def __await__(self): while True: r = self.poll(w.queue) if r is not _PENDING: - return r + return _typing.cast(_T, r) parked = w.parked[token] = loop.create_future() yield from parked finally: @@ -34,12 +34,12 @@ class _Wakeup: __slots__ = ("queue", "parked", "wakers", "pump", "__weakref__") - def __init__(self, loop): + def __init__(self, loop: _asyncio.AbstractEventLoop) -> None: self.queue = _n.Queue(loop) # The future each parked poll waits on, and each `Waker`'s callback, by token. - self.parked = {} - self.wakers = {} - self.pump = None + self.parked: dict[int, _asyncio.Future[None]] = {} + self.wakers: dict[int, _typing.Callable[[], object]] = {} + self.pump: _asyncio.Task[None] | None = None fd = self.queue.fileno() try: loop.add_reader(fd, self.dispatch, True) @@ -49,14 +49,14 @@ def __init__(self, loop): sock.setblocking(False) self.pump = loop.create_task(self._pump(loop, sock)) - async def _pump(self, loop, sock): + async def _pump(self, loop: _asyncio.AbstractEventLoop, sock: _socket.socket) -> None: with sock: while True: await loop.sock_recv(sock, 64) self.dispatch(True) - def dispatch(self, read): - loop = None + def dispatch(self, read: bool) -> None: + loop: _asyncio.AbstractEventLoop | None = None for token in self.queue.take(read): if token < 0: self.wakers.pop(-token, None) @@ -71,10 +71,10 @@ def dispatch(self, read): loop.call_soon(wake) -_wakeups = _weakref.WeakKeyDictionary() +_wakeups: _weakref.WeakKeyDictionary[_asyncio.AbstractEventLoop, _Wakeup] = _weakref.WeakKeyDictionary() -def _resume(): +def _resume() -> _n.Queue | None: """Called once a call on the running loop's thread returns having woken some of its futures or wakers: schedule their dispatch as the loop's next callback, never inside the call, and return the queue it takes from.""" @@ -86,7 +86,7 @@ def _resume(): return w.queue -def _wakeup(loop): +def _wakeup(loop: _asyncio.AbstractEventLoop) -> _Wakeup: w = _wakeups.get(loop) if w is None: w = _wakeups[loop] = _Wakeup(loop) @@ -100,7 +100,7 @@ class Waker: __slots__ = ("_node",) - def __init__(self, wake, loop=None): + def __init__(self, wake: _typing.Callable[[], object], loop: _asyncio.AbstractEventLoop | None = None) -> None: w = _wakeup(_get_running_loop() if loop is None else loop) self._node = _n.Node(w.queue) w.wakers[self._node.token()] = wake @@ -111,8 +111,8 @@ class _Const: __slots__ = ("get",) - def __init__(self, get): + def __init__(self, get: _typing.Callable[[], object]) -> None: self.get = get - def __get__(self, obj, cls=None): + def __get__(self, obj: object, cls: object = None) -> object: return self.get() diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 195caa1e2..1a87de1ba 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -30,6 +30,7 @@ use crate::ir::{Access, Family, Function, Int, Literal, Receiver, Traits, Type}; use crate::kotlin::{indent_by, pascal, rust_c, strip_refs}; const RUNTIME: &str = include_str!("../runtime/rt.py"); +const NATIVE: &str = include_str!("../runtime/native.py"); pub const GLUE_RUNTIME: &str = include_str!("../runtime/python.rs"); /// What the generator writes: the Python module, and the Rust module of the extension it @@ -45,6 +46,7 @@ pub struct Output { const TAKEN: &[&str] = &[ "_n", "_load", + "_T", "_Future", "_Wakeup", "_wakeups", @@ -71,6 +73,13 @@ const TAKEN: &[&str] = &[ "_AttributeError", ]; +/// Any buffer, as Python before 3.12's `collections.abc.Buffer` spells it. +const BUFFER: &str = "bytes | bytearray | memoryview"; + +/// A socket address as Python's `socket` module holds it: IPv4's, and IPv6's. +const V4: &str = "tuple[str, int]"; +const V6: &str = "tuple[str, int, int, int]"; + /// The registry's first entries, the runtime's: the glue's `T_FUTURE`, `T_TIMEDELTA`, and /// `T_RESUME`. const RUNTIME_TYPES: &[&str] = &["_Future", "_datetime.timedelta", "_resume"]; @@ -84,6 +93,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result, packages: &str, contract: u64) -> Result "\tpass\n".to_string(), + false => g + .consts + .iter() + .map(|c| format!("\t{}: {}\n{}", c.name, c.ty, docstring(&c.docs, "\t"))) + .collect(), + }; + // What only runs: the registry, and the module's constants, declared above for type + // checkers and made by `__getattr__` on each read. let mut body = format!( - "{RUNTIME}\n{}{}_n._register(\n{registry})\n\n_consts = {{\n{consts}}}\n\n\ndef __getattr__(name):\n\t\"\"\"A constant, made anew each time it is read, as Rust makes it.\"\"\"\n\tget = _consts.get(name)\n\tif get is None:\n\t\traise _AttributeError(f\"module {{__name__!r}} has no attribute {{name!r}}\")\n\treturn get()\n", + "{RUNTIME}\n{}{}if _typing.TYPE_CHECKING:\n{declared}else:\n\t_n._register(\n{registry}\t)\n\n\t_consts = {{\n{consts}\t}}\n\n\tdef __getattr__(name: str) -> object:\n\t\t\"\"\"A constant, made anew each time it is read, as Rust makes it.\"\"\"\n\t\tget = _consts.get(name)\n\t\tif get is None:\n\t\t\traise _AttributeError(f\"module {{__name__!r}} has no attribute {{name!r}}\")\n\t\treturn get()\n", g.py, g.top ); let mut hasher = StableHasher::new(); @@ -118,6 +138,8 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result, packages: &str, contract: u64) -> Result object:\n\ \timport importlib.machinery\n\ \timport importlib.util\n\ \timport os\n\n\ \tname = {{\"win32\": \"{p}.dll\", \"darwin\": \"lib{p}.dylib\"}}.get(_sys.platform, \"lib{p}.so\")\n\ \tpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), name)\n\ \tloader = importlib.machinery.ExtensionFileLoader(\"_{p}\", path)\n\ - \tmodule = importlib.util.module_from_spec(importlib.util.spec_from_loader(\"_{p}\", loader, origin=path))\n\ + \tmodule = importlib.util.module_from_spec(importlib.machinery.ModuleSpec(\"_{p}\", loader, origin=path))\n\ \tloader.exec_module(module)\n\ \treturn module\n\n\n\ - _n = _load()\n\ - # Checked before any other call: a Python file and a library from different builds\n\ - # must not mix.\n\ - if _n.contract() != 0x{contract:016x}:\n\ - \traise ImportError(\"{p}.py and the library are from different builds\")\n\ + if _typing.TYPE_CHECKING:\n\ + {native}{natives}\ + else:\n\ + \t_n = _load()\n\ + \t# Checked before any other call: a Python file and a library from different builds\n\ + \t# must not mix.\n\ + \tif _n.contract() != 0x{contract:016x}:\n\ + \t\traise ImportError(\"{p}.py and the library are from different builds\")\n\ _PENDING = _n.PENDING\n\n\n" ); body.insert_str(0, &head); @@ -233,12 +259,31 @@ struct Direct<'s> { docs: &'s str, } +/// A module-level constant that is no literal, made by its native function on each read. +struct Const { + name: String, + bare: String, + /// Its Python annotation. + ty: String, + docs: String, +} + +/// A function's Python form. +enum Form { + /// Python's own definition. + Python(String), + /// The glue's, declared for type checkers, and the statement binding it at run time + /// unless `_register` sets it on its class. + Native(String, Option), +} + /// A class's members: those Python defines, and those the glue does, which the class -/// declares only for type checkers. +/// declares only for type checkers and binds in `runtime` unless `_register` does. #[derive(Default)] struct Body { python: Vec, native: Vec, + runtime: Vec, } impl Body { @@ -253,6 +298,10 @@ impl Body { let native: Vec = self.native.iter().map(|m| indent_by(m, 1)).collect(); members.push(format!("\tif _typing.TYPE_CHECKING:\n{}", native.join("\n"))); } + if !self.runtime.is_empty() { + let runtime: Vec = self.runtime.iter().map(|m| indent_by(m, 1)).collect(); + members.push(format!("\telse:\n{}", runtime.join(""))); + } members } } @@ -262,8 +311,10 @@ struct Gen<'a> { /// The classes, then the module-level functions. py: String, top: String, - /// The module-level constants that are no literal, by name, with their native function. - consts: Vec<(String, String)>, + /// The module-level constants that are no literal. + consts: Vec, + /// The glue's functions the Python calls by name, declared for type checkers. + natives: Vec, glue: String, /// The extension's method table entries. table: Vec, @@ -351,7 +402,11 @@ impl<'a> Gen<'a> { Type::Char | Type::Str | Type::String | Type::Path | Type::PathBuf => "str".to_string(), Type::Slice | Type::Vec | Type::Bytes => "memoryview".to_string(), Type::Duration => "_datetime.timedelta".to_string(), - Type::SocketAddr(_) => "tuple".to_string(), + Type::SocketAddr(family) => match family { + Family::V4 => V4.to_string(), + Family::V6 => V6.to_string(), + Family::Any => format!("{V4} | {V6}"), + }, Type::List(inner) => format!("list[{}]", self.py_ty(inner)), Type::Waker => "Waker".to_string(), Type::Option(inner) => format!("{} | None", self.py_ty(inner)), @@ -361,8 +416,8 @@ impl<'a> Gen<'a> { Type::Object(i) => self.class(&abi.object(*i)), Type::Callback(i, _) => self.class(&abi.callback(*i)), Type::Closure(params, ret) => { - let params: Vec = params.iter().map(|p| self.py_param(p)).collect(); - format!("_typing.Callable[[{}], {}]", params.join(", "), self.py_ty(ret)) + let params: Vec = params.iter().map(|p| self.py_lent(p)).collect(); + format!("_typing.Callable[[{}], {}]", params.join(", "), self.py_param(ret)) } Type::Range(_) => "range".to_string(), Type::Bound(_) | Type::Bounds(_) => self.class(&abi.range(ty)), @@ -373,12 +428,14 @@ impl<'a> Gen<'a> { } } - /// The Python annotation of a parameter of type `ty`, either way: its bytes are any - /// buffer, and its path any path. + /// The Python annotation of a parameter of type `ty`, or a record's field, either way: + /// its bytes are any buffer, and its path any path. fn py_param(&self, ty: &Type) -> String { match ty { - Type::Slice | Type::Vec | Type::Bytes => "bytes".to_string(), + Type::Slice | Type::Vec | Type::Bytes => BUFFER.to_string(), Type::Path | Type::PathBuf => "str".to_string(), + // Python's own IPv6 address may leave out its flowinfo and scope_id. + Type::SocketAddr(Family::V6) => format!("{V4} | {V6}"), Type::List(inner) => format!("list[{}]", self.py_param(inner)), Type::Option(inner) => format!("{} | None", self.py_param(inner)), Type::Box(inner) | Type::Ref(_, inner) => self.py_param(inner), @@ -564,12 +621,11 @@ impl<'a> Gen<'a> { let (mut declared, mut bound) = (Vec::new(), Vec::new()); for function in &api.functions { match self.function(function, None, None, Place::Top) { - Some((code, true)) => { - let name = member_name(&code).expect("a function's name"); - bound.push(format!("\t{name} = _n.{name}\n")); + Some(Form::Native(code, binding)) => { + bound.push(indent_by(&binding.expect("a module's function is bound"), 1)); declared.push(indent_by(&code, 1)); } - Some((code, false)) => { + Some(Form::Python(code)) => { self.top.push_str(&code); self.top.push_str("\n\n"); } @@ -588,7 +644,7 @@ impl<'a> Gen<'a> { let py = self.py_ty(ty); let _ = write!( self.py, - "class {name}(_Exception):\n\t\"\"\"A `{py}` Rust returned as an error, in `error`, whose `str` is the exception's.\"\"\"\n\n\tdef __init__(self, error: {py}):\n\t\t_Exception.__init__(self, error)\n\t\tself.error = error\n\n\tdef __str__(self):\n\t\treturn _str(self.error)\n\n\n" + "class {name}(_Exception):\n\t\"\"\"A `{py}` Rust returned as an error, in `error`, whose `str` is the exception's.\"\"\"\n\n\tdef __init__(self, error: {py}) -> None:\n\t\t_Exception.__init__(self, error)\n\t\tself.error = error\n\n\tdef __str__(self) -> str:\n\t\treturn _str(self.error)\n\n\n" ); } } @@ -752,8 +808,12 @@ impl<'a> Gen<'a> { "let this = {this}?;\n\t\tlet other = {other}?;\n\t\tlet ret = rt::call(|| crate::{c}_eq(this, other));\n\t\ts.done();\n\t\trt::bool_out(ret)" ), ); + // Python makes a class that defines `__eq__` alone unhashable, as `Eq` without `Hash` is. body.python.push(format!( - "\tdef __eq__(self, other):\n\t\treturn _isinstance(other, {class}) and _n.{bare}_eq(self, other)\n\n\t__hash__ = None\n" + "\tdef __eq__(self, other: object) -> bool:\n\t\treturn _isinstance(other, {class}) and _n.{bare}_eq(self, other)\n" + )); + self.natives.push(format!( + "@_staticmethod\ndef {bare}_eq(this: {class}, other: {class}, /) -> bool: ...\n" )); } if t.default { @@ -799,11 +859,13 @@ impl<'a> Gen<'a> { Some(_) => Place::Member, None => Place::Static, }; - if let Some((code, native)) = self.function(method, Some(owner), Some((ty, &reserved)), place) { - match native { - true => body.native.push(indent_by(&code, 1)), - false => body.python.push(indent_by(&code, 1)), + match self.function(method, Some(owner), Some((ty, &reserved)), place) { + Some(Form::Native(code, binding)) => { + body.native.push(indent_by(&code, 1)); + body.runtime.extend(binding.map(|b| indent_by(&b, 1))); } + Some(Form::Python(code)) => body.python.push(indent_by(&code, 1)), + None => {} } } self.distinct(rust, data, body.codes()); @@ -842,7 +904,7 @@ impl<'a> Gen<'a> { format!( "\t{}: {}\n{}", field_name(&f.name), - self.py_ty(&f.ty), + self.py_param(&f.ty), docstring(&f.docs, "\t") ) }) @@ -922,6 +984,12 @@ impl<'a> Gen<'a> { .collect(); data.extend(fields); self.methods(&e.methods, (&ty, &owner, &rust), &data, &[], &mut body); + // Each variant's class is bound on the enum's once made, after the enum's own. + for variant in &e.variants { + let vname = pascal(&abi::snake(&variant.name)); + body.native + .push(format!("\t{vname}: _typing.TypeAlias = \"_{name}_{vname}\"\n")); + } let _ = write!( self.py, "class {name}:\n{}\t__slots__ = ()\n{}\n\n", @@ -942,7 +1010,7 @@ impl<'a> Gen<'a> { let fields: String = variant .fields .iter() - .map(|f| format!("\t{}: {}\n", field_name(&f.name), self.py_ty(&f.ty))) + .map(|f| format!("\t{}: {}\n", field_name(&f.name), self.py_param(&f.ty))) .collect(); let fields = match fields.is_empty() { true => "\tpass\n".to_string(), @@ -950,7 +1018,7 @@ impl<'a> Gen<'a> { }; let _ = write!( self.py, - "@_dataclass(slots=True{repr})\nclass {class}({name}):\n{fields}\n\n{name}.{vname} = {class}\n{class}.__name__ = \"{vname}\"\n{class}.__qualname__ = \"{name}.{vname}\"\n\n\n" + "@_dataclass(slots=True{repr})\nclass {class}({name}):\n{fields}\n\n{class}.__name__ = \"{vname}\"\n{class}.__qualname__ = \"{name}.{vname}\"\nif not _typing.TYPE_CHECKING:\n\t{name}.{vname} = {class}\n\n\n" ); match variant.fields.is_empty() { true => { @@ -1001,7 +1069,7 @@ impl<'a> Gen<'a> { let ty = Type::Object(i); let mut body = Body { python: vec!["\t__slots__ = ()\n".to_string()], - native: Vec::new(), + ..Body::default() }; self.traits(&ty, &c, object.traits, &mut body); let owner = Ty::Handle { @@ -1044,7 +1112,7 @@ impl<'a> Gen<'a> { body.push(format!( "\t@_abc.abstractmethod\n\tdef {member}({}) -> {}:\n{}\t\t...\n", params.join(", "), - self.py_ty(&method.ok), + self.py_param(&method.ok), docstring(&method.docs, "\t\t") )); let c_params: Vec = method.params.iter().map(|p| abi.ty(&p.ty)).collect(); @@ -1208,7 +1276,7 @@ impl<'a> Gen<'a> { owner: Option<&Ty>, owner_ty: Option<(&Type, &[&str])>, place: Place, - ) -> Option<(String, bool)> { + ) -> Option
{ let abi = self.abi; let export = abi.export(function, owner); let c_name = export.call.name.clone(); @@ -1230,6 +1298,8 @@ impl<'a> Gen<'a> { let mut c_args = Vec::new(); let mut py_params = Vec::new(); let mut py_args = Vec::new(); + // Its native function's parameters, declared for type checkers. + let mut native_params = Vec::new(); let mut n = 0; let mut c_params = export.call.params.iter(); let receiver = function.receiver.filter(|_| owner.is_some()); @@ -1238,6 +1308,7 @@ impl<'a> Gen<'a> { let ty = owner_ty.expect("a method has its type").0.clone(); py_params.push("self".to_string()); py_args.push("self".to_string()); + native_params.push(format!("this: {}", self.py_ty(&ty))); match c { Ty::Handle { .. } => { let ty = match receiver { @@ -1260,6 +1331,7 @@ impl<'a> Gen<'a> { let (_, c) = c_params.next().expect("each parameter has its C one"); let name = param_name(¶m.name); py_params.push(format!("{name}: {}", self.py_param(¶m.ty))); + native_params.push(format!("{name}: {}", self.py_param(¶m.ty))); py_args.push(name.clone()); match (¶m.ty, c) { (Type::Closure(lent, ret), Ty::Fn(c_lent, c_ret)) => { @@ -1378,9 +1450,18 @@ impl<'a> Gen<'a> { let params = py_params.join(", "); // The glue's own is only declared, for type checkers. let code = match (export.future.is_some(), function.eager) { - (true, false) => format!( - "async def {member}({params}) -> {ret}:\n{docs}\tr = {native}\n\treturn (await r) if r.__class__ is _Future else r" - ), + (true, false) => { + if !native_params.is_empty() { + native_params.push("/".to_string()); + } + self.natives.push(format!( + "@_staticmethod\ndef {bare}({}) -> {ret} | _Future[{ret}]: ...\n", + native_params.join(", ") + )); + format!( + "async def {member}({params}) -> {ret}:\n{docs}\tr = {native}\n\treturn (await r) if _isinstance(r, _Future) else r" + ) + } (true, true) => format!("def {member}({params}) -> _typing.Awaitable[{ret}]:\n{docs}\t..."), (false, _) => format!("def {member}({params}) -> {ret}:\n{docs}\t..."), }; @@ -1388,15 +1469,27 @@ impl<'a> Gen<'a> { (Access::Const, Place::Top) if constant => { let name = member.to_uppercase(); self.claim(&name, format!("const `{}`", function.path.join("::"))); - self.consts.push((name, bare)); + self.consts.push(Const { + name, + bare, + ty: ret, + docs: text, + }); return None; } (Access::Const, Place::Static) if constant => { - format!( - "{} = _Const(_n.{bare})\n{}", - member.to_uppercase(), - docstring(text.trim(), "") - ) + let name = member.to_uppercase(); + self.natives + .push(format!("@_staticmethod\ndef {bare}() -> {ret}: ...\n")); + // An `Enum` makes a member of any other value, to a type checker as at run time. + let value = match owner_ty { + Some((Type::Enum(i), _)) if abi.api.enums[*i].plain() => format!("_enum.nonmember(_n.{bare}())"), + _ => format!("_n.{bare}()"), + }; + return Some(Form::Native( + format!("{name} = {value}\n{}", docstring(text.trim(), "")), + Some(format!("{name} = _Const(_n.{bare})\n")), + )); } _ if property => format!("@_property\n{code}"), (_, Place::Static) => format!("@_staticmethod\n{code}"), @@ -1405,7 +1498,11 @@ impl<'a> Gen<'a> { if place == Place::Top { self.claim(&member, format!("fn `{}`", function.path.join("::"))); } - Some((code, direct)) + Some(match (direct, place) { + (true, Place::Top) => Form::Native(code, Some(format!("{member} = _n.{member}\n"))), + (true, _) => Form::Native(code, None), + (false, _) => Form::Python(code), + }) } /// The out values of `outs`, for a function returning `ok` or raising `err` as @@ -1537,10 +1634,8 @@ fn member_name(code: &str) -> Option { if let Some(rest) = line.strip_prefix("def ") { return Some(rest.chars().take_while(|c| *c != '(').collect()); } - if let Some((name, _)) = line.split_once(" =") { - return Some(name.split(':').next().unwrap_or(name).trim().to_string()); - } - None + let name: String = line.chars().take_while(|c| c.is_alphanumeric() || *c == '_').collect(); + (!name.is_empty()).then_some(name) } /// A C string literal of `text`. diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index b1887cb4f..a204d5a56 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -572,6 +572,22 @@ fn api_python() { ); } +/// The api fixture's Python passes mypy and pyright, both strict, at the oldest CPython +/// its extension loads in. Apart from `api_python`, which the runners without the dev +/// shell's checkers run alone. +#[test] +fn api_py_types() { + let api = api().out.join("python").join("api.py"); + run(Command::new("mypy") + .args(["--strict", "--python-version", "3.11", "--cache-dir"]) + .arg(scratch().join("mypy")) + .arg(&api)); + run(Command::new("pyright") + .arg("-p") + .arg(manifest_dir().join("tests").join("pyright.json")) + .arg(&api)); +} + /// The contract a Kotlin file checks its library against. fn kotlin_contract(kotlin: &Path) -> u64 { let kotlin = std::fs::read_to_string(kotlin).unwrap(); diff --git a/ffi/tests/pyright.json b/ffi/tests/pyright.json new file mode 100644 index 000000000..65d06cd5b --- /dev/null +++ b/ffi/tests/pyright.json @@ -0,0 +1,4 @@ +{ + "typeCheckingMode": "strict", + "pythonVersion": "3.11" +} diff --git a/flake.nix b/flake.nix index aa5bbc702..d2ad6a898 100644 --- a/flake.nix +++ b/flake.nix @@ -297,10 +297,12 @@ typescript bun esbuild - # rust2-ffi's tests compile its generated Kotlin and run it on the JVM, run its - # generated Python and Dart, and build its generated Go through cgo. + # rust2-ffi's tests compile its generated Kotlin and run it on the JVM, run and + # type-check its generated Python, run its Dart, and build its Go through cgo. kotlin python3 + mypy + pyright # rust2-go's tests build and run its output against native Rust. go dart diff --git a/quest/m1/ffi-export/README.md b/quest/m1/ffi-export/README.md index 6afb3de85..93188862a 100644 --- a/quest/m1/ffi-export/README.md +++ b/quest/m1/ffi-export/README.md @@ -64,5 +64,4 @@ names and defaults, and fails the export on a public item it cannot export. - [C names and crates with no config](/quest/m1/ffi-export/names.md) - a collision-free naming rule, a facade crate, and the prefix from Cargo, leaving only `exclude` in `ffi.toml` - [Sessions on sans-IO QUIC](/quest/m1/ffi-export/session-quic.md) - bindings move UDP datagrams and drive the session's poll themselves, over moq's runtime-free QUIC driver once it lands - [C++ bindings on MSVC](/quest/m1/ffi-export/cpp.md) - the nightly's MSVC job passes the C++ checks -- [Generated Python type-checks](/quest/m1/ffi-export/python-types.md) - mypy and pyright pass on the generated Python - [Every public item exports](/quest/m1/ffi-export/refusals.md) - the census is empty and `ffi.toml` is deleted, so the exporter reads no config diff --git a/quest/m1/ffi-export/python-types.md b/quest/m1/ffi-export/python-types.md deleted file mode 100644 index d6ba2bc1b..000000000 --- a/quest/m1/ffi-export/python-types.md +++ /dev/null @@ -1,16 +0,0 @@ -# [S] Generated Python type-checks - -## Goal - -The Python that [rust2-ffi](/ffi/README.md) generates passes mypy and pyright, checked in -CI on the api fixture and moq's export, so a typed consumer imports it -with no ignores. - -## Plan - -On `main` it fails both, at least on: classes based on the untyped `_n.Object`, enum -variants assigned as class attributes, `__hash__ = None` on unhashable classes, and the -extension loader's module spec. Fix the generator, not the checks; a construct no stub can -type is a gap in the generator's output, not a reason to ignore it. - -Public API: the generated Python's types, not its behaviour. From c615f90fca986b855b63f18c918ecdc0aa11e564 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:23:18 -0800 Subject: [PATCH 3/6] fix(ffi): name a native stub's parameters by position, so none clashes with its receiver Co-Authored-By: Claude Opus 5.5 --- ffi/src/python.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 1a87de1ba..a6b6fd1c0 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -1298,7 +1298,8 @@ impl<'a> Gen<'a> { let mut c_args = Vec::new(); let mut py_params = Vec::new(); let mut py_args = Vec::new(); - // Its native function's parameters, declared for type checkers. + // Its native function's parameters, declared for type checkers by position, so no + // Rust name clashes with the receiver's. let mut native_params = Vec::new(); let mut n = 0; let mut c_params = export.call.params.iter(); @@ -1308,7 +1309,7 @@ impl<'a> Gen<'a> { let ty = owner_ty.expect("a method has its type").0.clone(); py_params.push("self".to_string()); py_args.push("self".to_string()); - native_params.push(format!("this: {}", self.py_ty(&ty))); + native_params.push(format!("p0: {}", self.py_ty(&ty))); match c { Ty::Handle { .. } => { let ty = match receiver { @@ -1331,7 +1332,7 @@ impl<'a> Gen<'a> { let (_, c) = c_params.next().expect("each parameter has its C one"); let name = param_name(¶m.name); py_params.push(format!("{name}: {}", self.py_param(¶m.ty))); - native_params.push(format!("{name}: {}", self.py_param(¶m.ty))); + native_params.push(format!("p{}: {}", native_params.len(), self.py_param(¶m.ty))); py_args.push(name.clone()); match (¶m.ty, c) { (Type::Closure(lent, ret), Ty::Fn(c_lent, c_ret)) => { From b651d914a7a739262ca6380e91cf4d6893eadeea Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:39:09 -0800 Subject: [PATCH 4/6] docs(ffi): rewrap the Python section's lines Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index a01ffee1d..f9df970fe 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -336,14 +336,14 @@ caller imports it with no ignores. `Event.Moved(p)`. `Display` is `str()`, `Debug` `repr()`, and `Default` a static `default()`. Constants are module or class attributes, made anew on each read unless a literal. -- **Bytes** passed in are any buffer, `bytes | bytearray | memoryview`: `bytes` is read in place, and others, which Python - may change while Rust reads, are copied. Bytes Rust returns, or a record holds, are a - read-only `memoryview` lending Rust's buffer until it and its views are released. Those - Rust lends a closure or trait method are copied into `bytes`. Strings are `str`, - `Option` `T | None`, a tuple a `tuple`, `Duration` `datetime.timedelta`, truncated to - the microsecond, with `timedelta.max` for any longer and for `Duration::MAX`, a `Range` a - `range`, and an `Err` a raised `Exception` holding it in `error`, whose `str` is its - `Display`. +- **Bytes** passed in are any buffer, `bytes | bytearray | memoryview`: `bytes` is read in + place, and others, which Python may change while Rust reads, are copied. Bytes Rust + returns, or a record holds, are a read-only `memoryview` lending Rust's buffer until it + and its views are released. Those Rust lends a closure or trait method are copied into + `bytes`. Strings are `str`, `Option` `T | None`, a tuple a `tuple`, `Duration` + `datetime.timedelta`, truncated to the microsecond, with `timedelta.max` for any longer + and for `Duration::MAX`, a `Range` a `range`, and an `Err` a raised `Exception` + holding it in `error`, whose `str` is its `Display`. - A **path** is a `str`, or any path passed in, which crosses as `os.fsencode` encodes it, so one that is not UTF-8 keeps its bytes. A `SocketAddr` is the `socket` module's tuple, `(host, port)` or `(host, port, flowinfo, scope_id)`, of that family for `V4` or `V6`. @@ -572,8 +572,8 @@ agree; the runtime the glue includes is [`runtime/rt.rs`](runtime/rt.rs), the on header includes [`runtime/rt.hpp`](runtime/rt.hpp), the ones Kotlin and its JNI glue include [`runtime/rt.kt`](runtime/rt.kt) and [`runtime/jni.rs`](runtime/jni.rs), the ones Python and its extension include [`runtime/rt.py`](runtime/rt.py), -[`runtime/native.py`](runtime/native.py), and [`runtime/python.rs`](runtime/python.rs), the one the Swift includes -[`runtime/rt.swift`](runtime/rt.swift), the one the Go includes +[`runtime/native.py`](runtime/native.py), and [`runtime/python.rs`](runtime/python.rs), +the one the Swift includes [`runtime/rt.swift`](runtime/rt.swift), the one the Go includes [`runtime/rt.go`](runtime/rt.go), and the ones the Dart and its glue include [`runtime/rt.dart`](runtime/rt.dart) and [`runtime/dart.rs`](runtime/dart.rs). From 3470fc443de603746d4ddaab6738963f251707f4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:42:21 -0800 Subject: [PATCH 5/6] test(ffi): an async method whose parameter is named this Gauge::reaches takes `this`, which the Python's native declarations once also named the receiver: a duplicate argument, a SyntaxError on import. Co-Authored-By: Claude Opus 5.5 --- ffi/fixtures/api/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index 49498c93e..5662a4c6a 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -800,6 +800,13 @@ impl Gauge { yielded(0).await; self.level() } + + /// Whether the level reaches `this`, read once [`yielded`] finishes: a parameter named + /// as a binding might name the receiver. + pub async fn reaches(&self, this: u64) -> bool { + yielded(0).await; + self.level() >= this + } } /// The gauge lent shared to `watch` for the call. From 2aadf0411cbaf50e83d10c0d0dbc7f94c9992dc2 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:33:25 -0800 Subject: [PATCH 6/6] fix(ffi): annotate Python list inputs as covariant containers A list parameter was the invariant `list[...]` of its widened element, so a caller's `list[tuple[str, int]]` passed to `ports` failed mypy and pyright. Parameters, and what Python returns to Rust, are now `Iterable[...]`, as `rt::list_in` reads any iterable; a record's list field is `Sequence[...]`, since the record keeps it and Rust may read it again. `api_py_types` now also type-checks `tests/api_types.py`, a typed caller passing such lists. Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 4 +++- ffi/src/python.rs | 30 ++++++++++++++++++++++-------- ffi/tests/api_types.py | 17 +++++++++++++++++ ffi/tests/fixtures.rs | 24 +++++++++++++++--------- 4 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 ffi/tests/api_types.py diff --git a/ffi/README.md b/ffi/README.md index f9df970fe..7594b3696 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -347,7 +347,9 @@ caller imports it with no ignores. - A **path** is a `str`, or any path passed in, which crosses as `os.fsencode` encodes it, so one that is not UTF-8 keeps its bytes. A `SocketAddr` is the `socket` module's tuple, `(host, port)` or `(host, port, flowinfo, scope_id)`, of that family for `V4` or `V6`. - A `Vec` or `&[T]` is a `list`, and passing a `Vec` by value closes each object in it. + A `Vec` or `&[T]` Rust returns is a `list`, and passed in is any iterable other than a `str`, + annotated `Iterable`, or `Sequence` in a record, which keeps it. Passing a `Vec` by value closes + each object in it. An integer out of its Rust type's range raises `OverflowError`. - **Traits** are abstract classes an implementation subclasses; once Rust drops one, its `close()` is called if it has one. diff --git a/ffi/src/python.rs b/ffi/src/python.rs index a6b6fd1c0..85fc86f45 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -428,19 +428,33 @@ impl<'a> Gen<'a> { } } - /// The Python annotation of a parameter of type `ty`, or a record's field, either way: - /// its bytes are any buffer, and its path any path. + /// The Python annotation of a parameter of type `ty`, or of what Python returns to Rust: + /// its bytes are any buffer, its path any path, and its list any iterable, which + /// `rt::list_in` reads once. fn py_param(&self, ty: &Type) -> String { + self.py_in(ty, "_typing.Iterable") + } + + /// The Python annotation of a record's field of type `ty`: as a parameter's, but its list + /// is a sequence, since the record keeps it and Rust may read it more than once. + fn py_field(&self, ty: &Type) -> String { + self.py_in(ty, "_typing.Sequence") + } + + /// The Python annotation of a value of type `ty` that Python passes in, each list as the + /// covariant container `list` names, so a caller's `list` of one alternative of a widened + /// element type checks. + fn py_in(&self, ty: &Type, list: &str) -> String { match ty { Type::Slice | Type::Vec | Type::Bytes => BUFFER.to_string(), Type::Path | Type::PathBuf => "str".to_string(), // Python's own IPv6 address may leave out its flowinfo and scope_id. Type::SocketAddr(Family::V6) => format!("{V4} | {V6}"), - Type::List(inner) => format!("list[{}]", self.py_param(inner)), - Type::Option(inner) => format!("{} | None", self.py_param(inner)), - Type::Box(inner) | Type::Ref(_, inner) => self.py_param(inner), + Type::List(inner) => format!("{list}[{}]", self.py_in(inner, list)), + Type::Option(inner) => format!("{} | None", self.py_in(inner, list)), + Type::Box(inner) | Type::Ref(_, inner) => self.py_in(inner, list), Type::Tuple(elements) => { - let elements: Vec = elements.iter().map(|e| self.py_param(e)).collect(); + let elements: Vec = elements.iter().map(|e| self.py_in(e, list)).collect(); format!("tuple[{}]", elements.join(", ")) } _ => self.py_ty(ty), @@ -904,7 +918,7 @@ impl<'a> Gen<'a> { format!( "\t{}: {}\n{}", field_name(&f.name), - self.py_param(&f.ty), + self.py_field(&f.ty), docstring(&f.docs, "\t") ) }) @@ -1010,7 +1024,7 @@ impl<'a> Gen<'a> { let fields: String = variant .fields .iter() - .map(|f| format!("\t{}: {}\n", field_name(&f.name), self.py_param(&f.ty))) + .map(|f| format!("\t{}: {}\n", field_name(&f.name), self.py_field(&f.ty))) .collect(); let fields = match fields.is_empty() { true => "\tpass\n".to_string(), diff --git a/ffi/tests/api_types.py b/ffi/tests/api_types.py new file mode 100644 index 000000000..1f9958d57 --- /dev/null +++ b/ffi/tests/api_types.py @@ -0,0 +1,17 @@ +# A typed caller of the api fixture's Python, which tests/fixtures.rs's `api_py_types` +# type-checks alongside it but never runs. Each list it passes in holds one alternative of +# a widened element type, or is no `list` at all, as a caller's own may be. + +import api + +v4s: list[tuple[str, int]] = [("127.0.0.1", 8080)] +ports: list[int] = api.ports(v4s) +ports = api.ports(addr for addr in v4s) + +rows: list[list[int]] = [[1, 2], [3]] +sums: list[int] = api.sums(rows) +sums = api.sums(((1, 2), (3,))) + +names: tuple[str, ...] = ("a", "b") +peer = api.Peer(v4s[0], names) +first: str = peer.names[0] diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index a204d5a56..446343a80 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -572,20 +572,26 @@ fn api_python() { ); } -/// The api fixture's Python passes mypy and pyright, both strict, at the oldest CPython -/// its extension loads in. Apart from `api_python`, which the runners without the dev -/// shell's checkers run alone. +/// The api fixture's Python, and `tests/api_types.py` calling it as a typed caller does, +/// pass mypy and pyright, both strict, at the oldest CPython its extension loads in. Apart +/// from `api_python`, which the runners without the dev shell's checkers run alone. #[test] fn api_py_types() { - let api = api().out.join("python").join("api.py"); + let dir = scratch().join("api").join("python-types"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let module = dir.join("api.py"); + let caller = dir.join("api_types.py"); + std::fs::copy(api().out.join("python").join("api.py"), &module).unwrap(); + std::fs::copy(manifest_dir().join("tests").join("api_types.py"), &caller).unwrap(); run(Command::new("mypy") .args(["--strict", "--python-version", "3.11", "--cache-dir"]) .arg(scratch().join("mypy")) - .arg(&api)); - run(Command::new("pyright") - .arg("-p") - .arg(manifest_dir().join("tests").join("pyright.json")) - .arg(&api)); + .args([&module, &caller])); + // Rooted in `dir`, so `import api` finds the module rather than `tests/api.py`. + let config = dir.join("pyrightconfig.json"); + std::fs::copy(manifest_dir().join("tests").join("pyright.json"), &config).unwrap(); + run(Command::new("pyright").arg("-p").arg(&config).args([&module, &caller])); } /// The contract a Kotlin file checks its library against.