diff --git a/.github/requirements-capi.txt b/.github/requirements-capi.txt new file mode 100644 index 00000000..02c7264f --- /dev/null +++ b/.github/requirements-capi.txt @@ -0,0 +1,4 @@ +# Python packages for the GOPY_BACKEND=capi job in workflows/ci.yml. +# pybindgen is deliberately absent: the capi backend must not need it. +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/requirements-cffi.txt b/.github/requirements-cffi.txt new file mode 100644 index 00000000..aff0e708 --- /dev/null +++ b/.github/requirements-cffi.txt @@ -0,0 +1,5 @@ +# Python packages for the GOPY_BACKEND=cffi job in workflows/ci.yml. +# pybindgen is deliberately absent: the cffi backend must not need it. +cffi +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/requirements-nanobind.txt b/.github/requirements-nanobind.txt new file mode 100644 index 00000000..130ee127 --- /dev/null +++ b/.github/requirements-nanobind.txt @@ -0,0 +1,5 @@ +# Python packages for the GOPY_BACKEND=nanobind job in workflows/ci.yml. +# pybindgen is deliberately absent: the nanobind backend must not need it. +nanobind +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/requirements-pybind11.txt b/.github/requirements-pybind11.txt new file mode 100644 index 00000000..4656bd43 --- /dev/null +++ b/.github/requirements-pybind11.txt @@ -0,0 +1,5 @@ +# Python packages for the GOPY_BACKEND=pybind11 job in workflows/ci.yml. +# pybindgen is deliberately absent: the pybind11 backend must not need it. +pybind11 +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af0017d8..bce4da07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,3 +96,107 @@ jobs: - name: Upload-Coverage if: matrix.platform == 'ubuntu-latest' uses: codecov/codecov-action@v4 + + # Builds and tests each opt-in backend (GOPY_BACKEND), beside the main + # matrix and on one Go version. None installs pybindgen: no backend here + # may need it. + # cffi: loads the cgo shim at runtime, with no C compiler step. + # pybind11: compiles a C++ module against the same shim (see noAPIShim in + # bind/noapi.go), so needs no extra system packages: a C++ + # compiler is already required for CGO_ENABLED=1 everywhere. + # nanobind: as pybind11 (see buildCXXModule in cmd_build.go), and also + # compiles nanobind's own runtime (nb_combined.cpp) into every + # module. + # capi: builds exactly as the default backend does, except that + # build.py writes the C module itself (see bind/capi_build.py). + backends: + name: ${{ matrix.backend }} backend (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + strategy: + # a failure in one backend doesn't cancel the others + fail-fast: false + matrix: + backend: [cffi, pybind11, nanobind, capi] + platform: [ubuntu-latest, windows-latest, macos-15] + python-version: ['3.11', '3.12'] + include: + # CXX defaults to "c++" in buildCXXModule (cmd_build.go), which + # isn't a recognized command on windows-latest's MinGW toolchain (the + # same one CGO_ENABLED=1 already needs there, so no extra install is + # needed). timeout raises go test's own default (10m): each test + # here compiles twice (cgo, then a separate C++ step), and once + # measured at ~18.5s/test average on windows-latest, 32 tests alone + # used 590s of the default budget. + - backend: pybind11 + cxx: g++ + timeout: 30m + - backend: nanobind + cxx: g++ + timeout: 30m + runs-on: ${{ matrix.platform }} + env: + GOPY_BACKEND: ${{ matrix.backend }} + CXX: ${{ matrix.cxx }} + # print the python stack if the process crashes, e.g. at exit + PYTHONFAULTHANDLER: 1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: .github/requirements-${{ matrix.backend }}.txt + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache: true + + - name: Install packages + run: | + python -m pip install -r .github/requirements-${{ matrix.backend }}.txt + go install golang.org/x/tools/cmd/goimports@v0.29.0 + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v -timeout=${{ matrix.timeout || '10m' }} ./... + + # Compares per-call overhead across backends (see _examples/bench/run.sh): + # not a pass/fail check, just a table uploaded as a build artifact. Runs + # on ubuntu-latest only -- the comparison is between backends, not OSes. + benchmark: + name: benchmark backends + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache: true + + - name: Install packages + run: | + python -m pip install pybindgen cffi pybind11 nanobind + go install golang.org/x/tools/cmd/goimports@v0.29.0 + + - name: Run benchmark + run: bash _examples/bench/run.sh | tee benchmark.txt + + - name: Upload benchmark table + uses: actions/upload-artifact@v4 + with: + name: benchmark + path: benchmark.txt diff --git a/README.md b/README.md index 2eec0f17..0b362b45 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ New features: Gopy now assumes that you are working with modules-based builds, and requires a valid `go.mod` file, and works only with Go versions 1.15 and above. -Currently using [pybindgen](https://pybindgen.readthedocs.io/en/latest/tutorial/) to generate the low-level c-to-python bindings, but support for [cffi](https://cffi.readthedocs.io/en/latest/) should be relatively straightforward for those using PyPy instead of CPython (pybindgen should be significantly faster for CPython apparently). You also need `goimports` to ensure the correct imports are included. +By default, gopy uses [pybindgen](https://pybindgen.readthedocs.io/en/latest/tutorial/) to generate the low-level c-to-python bindings. You also need `goimports` to ensure the correct imports are included. ```sh $ python3 -m pip install pybindgen @@ -29,6 +29,22 @@ $ go install github.com/go-python/gopy@latest (This all assumes you have already installed [Go itself](https://golang.org/doc/install), and added `~/go/bin` to your `PATH`). +### Choosing a backend (experimental) + +The `GOPY_BACKEND` environment variable picks a different tool for those bindings, for `gopy gen`, `gopy build` and `gopy pkg`: + +| `GOPY_BACKEND` | Needs | | +|---|---|---| +| `pybindgen` (default) | `pip install pybindgen` | | +| `capi` | nothing beyond python itself | writes the C-API bindings directly, without pybindgen | +| `cffi` | `pip install cffi` | | +| `pybind11` | `pip install pybind11`, a C++ compiler | | +| `nanobind` | `pip install nanobind`, a C++ compiler | | + +```sh +$ GOPY_BACKEND=capi gopy build github.com/go-python/gopy/_examples/hi +``` + To [install python modules](https://packaging.python.org/tutorials/packaging-projects/), you will need the python install packages: ```sh diff --git a/SUPPORT_MATRIX.md b/SUPPORT_MATRIX.md index 8f30be77..afaae308 100644 --- a/SUPPORT_MATRIX.md +++ b/SUPPORT_MATRIX.md @@ -6,6 +6,7 @@ don't modify manually. Feature |py3 --- | --- _examples/arrays | yes +_examples/callbacks | yes _examples/cgo | yes _examples/consts | yes _examples/cstrings | yes diff --git a/_examples/bench/bench.go b/_examples/bench/bench.go new file mode 100644 index 00000000..1599d75f --- /dev/null +++ b/_examples/bench/bench.go @@ -0,0 +1,23 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bench has a few trivial functions used to compare the per-call +// overhead of gopy's backends (GOPY_BACKEND); see bench.py and the +// "benchmark" job in .github/workflows/ci.yml. Not part of the test suite +// itself: no _examples/bench entry in main_test.go's features map. +package bench + +// Add returns the sum of its arguments: the cheapest possible call, to +// isolate per-call FFI overhead from any argument-marshaling cost. +func Add(i, j int) int { + return i + j +} + +// Concat concatenates two strings: a second data point, since string +// arguments/returns cross the boundary very differently from an int +// (a managed buffer + length or a null-terminated copy, depending on the +// backend) and might not share the int case's relative cost. +func Concat(a, b string) string { + return a + b +} diff --git a/_examples/bench/run.sh b/_examples/bench/run.sh new file mode 100755 index 00000000..dc25fb12 --- /dev/null +++ b/_examples/bench/run.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Builds _examples/bench under each of gopy's backends and prints a table +# comparing run_bench.py's timing/memory numbers across them. Run from the repo +# root; needs pybindgen, cffi, pybind11 and nanobind all installed for the python +# interpreter named by $PYTHON (defaults to python3), and a C++ compiler. +set -eu + +PYTHON="${PYTHON:-python3}" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +echo "building gopy..." +GOPY="$WORK/gopy" +(cd "$REPO" && go build -o "$GOPY" .) + +printf '%-10s %8s %14s %14s %10s\n' backend calls "add (s)" "concat (s)" "peak (KB)" + +for backend in pybindgen capi cffi pybind11 nanobind; do + out="$WORK/$backend" + mkdir -p "$out" + # gopy build cds into -output and runs go build there; give it a module + # that resolves back to this checkout, same as go.mod already does for + # anyone building _examples/* in place. + printf 'module dummy\n\nrequire github.com/go-python/gopy v0.0.0\nreplace github.com/go-python/gopy => %s\n' "$REPO" >"$out/go.mod" + GOPY_BACKEND="$backend" "$GOPY" build -vm="$PYTHON" -output="$out" -no-make -package-prefix= \ + "$REPO/_examples/bench" >"$out/build.log" 2>&1 || { + echo "$backend: build failed, see $out/build.log" >&2 + tail -n 20 "$out/build.log" >&2 + continue + } + cp "$REPO/_examples/bench/run_bench.py" "$out/" + row="$(cd "$out" && "$PYTHON" run_bench.py)" + IFS=, read -r calls add_s concat_s peak_kb <<<"$row" + printf '%-10s %8s %14s %14s %10s\n' "$backend" "$calls" "$add_s" "$concat_s" "$peak_kb" +done diff --git a/_examples/bench/run_bench.py b/_examples/bench/run_bench.py new file mode 100644 index 00000000..a86f96b9 --- /dev/null +++ b/_examples/bench/run_bench.py @@ -0,0 +1,47 @@ +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Times N calls each of bench.Add and bench.Concat, and the peak memory +# after them, printing one CSV line: iterations,add_seconds,concat_seconds,peak_kb +# (see run.sh, which builds this example under each backend and prints the +# resulting rows as a table -- this script itself doesn't know which +# backend it was built with). + +from __future__ import print_function + +import sys +import time + +import bench + +if sys.platform == "win32": + import psutil + + def peak_kb(): + return psutil.Process().memory_info().rss // 1024 +else: + import resource + + def peak_kb(): + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + + +N = 1000 +WARMUP = 100 + +for i in range(WARMUP): + bench.Add(i, i) + bench.Concat("a", "b") + +start = time.perf_counter() +for i in range(N): + bench.Add(i, i) +add_seconds = time.perf_counter() - start + +start = time.perf_counter() +for i in range(N): + bench.Concat("a", "b") +concat_seconds = time.perf_counter() - start + +print("%d,%f,%f,%d" % (N, add_seconds, concat_seconds, peak_kb())) diff --git a/_examples/callbacks/callbacks.go b/_examples/callbacks/callbacks.go new file mode 100644 index 00000000..739bfa55 --- /dev/null +++ b/_examples/callbacks/callbacks.go @@ -0,0 +1,118 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package callbacks has Go functions that take Python callables, and call +// them before returning. +package callbacks + +import ( + "fmt" + "sync" + "time" +) + +// Each calls fun for i in 0..n-1, with a label made from i. +func Each(n int, fun func(i int, label string)) { + for i := 0; i < n; i++ { + fun(i, fmt.Sprintf("item-%d", i)) + } +} + +// Mixed calls fun with a bool, a float and an unsigned integer. +func Mixed(fun func(on bool, x float64, u uint8)) { + fun(true, 1.5, 200) + fun(false, -2.25, 7) +} + +// Twice calls fun, which takes no arguments, two times. +func Twice(fun func()) { + fun() + fun() +} + +// Describe calls fun with a string and a fmt.Stringer, as interface{} values. +// They arrive as strings, made by fmt.Sprintf("%s", v). +func Describe(fun func(v interface{})) { + fun("a string") + fun(1500 * time.Millisecond) +} + +// Count returns how many of 0..n-1 keep says yes to. +func Count(n int, keep func(i int) bool) int { + total := 0 + for i := 0; i < n; i++ { + if keep(i) { + total++ + } + } + return total +} + +// Sum adds up what val returns for 0..n-1. +func Sum(n int, val func(i int) int) int { + total := 0 + for i := 0; i < n; i++ { + total += val(i) + } + return total +} + +// Widest returns the largest of what size returns for 0..n-1. +func Widest(n int, size func(i int) uint) uint { + var widest uint + for i := 0; i < n; i++ { + if w := size(i); w > widest { + widest = w + } + } + return widest +} + +// Apply returns f(x). +func Apply(x float64, f func(x float64) float64) float64 { + return f(x) +} + +// Counter counts how many times it has been visited. +type Counter struct { + N int +} + +// Visit calls fun with the counter itself, which arrives as a handle. +func (c *Counter) Visit(times int, fun func(c *Counter, n int)) { + for i := 0; i < times; i++ { + c.N++ + fun(c, c.N) + } +} + +// Check calls fun with the counter itself, and reports what it answered. +func (c *Counter) Check(fun func(c *Counter, n int) bool) bool { + return fun(c, c.N) +} + +// InGoroutine calls fun from another goroutine, and waits for it. +func InGoroutine(fun func(i int)) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + fun(7) + }() + wg.Wait() +} + +var kept func(i int) + +// Keep stores fun, for CallKept to call after Keep has returned. +func Keep(fun func(i int)) { + kept = fun +} + +// CallKept calls the func Keep stored, outside the call it was passed to. +func CallKept(i int) { + if kept != nil { + kept(i) + } +} diff --git a/_examples/callbacks/test.py b/_examples/callbacks/test.py new file mode 100644 index 00000000..d315b19c --- /dev/null +++ b/_examples/callbacks/test.py @@ -0,0 +1,100 @@ +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from __future__ import print_function + +import io +import os +import subprocess +import sys + +import callbacks + +print("--- Each: int and string arguments") +callbacks.Each(3, lambda i, label: print("each:", i, label)) + +print("--- Mixed: bool, float and uint8 arguments") +# bool() because the pybindgen backend passes a bool as 1 or 0 +callbacks.Mixed(lambda on, x, u: print("mixed:", bool(on), x, u)) + +print("--- Twice: no arguments") +calls = [] +callbacks.Twice(lambda: calls.append(1)) +print("twice:", len(calls)) + +print("--- Counter.Visit: a Go struct arrives as a handle") +c = callbacks.Counter() + +def visit(handle, n): + seen = callbacks.Counter(handle=handle) + print("visit:", n, seen.N) + +c.Visit(2, visit) +print("counter:", c.N) + +print("--- Describe: an interface{} arrives as a string") +callbacks.Describe(lambda v: print("describe:", repr(v))) + +print("--- Count: a bool result") +print("count:", callbacks.Count(10, lambda i: i % 3 == 0)) + +print("--- Sum: an int result") +print("sum:", callbacks.Sum(5, lambda i: i * i)) + +print("--- Widest: a uint result") +print("widest:", callbacks.Widest(4, lambda i: i * 10)) + +print("--- Apply: a float result") +print("apply:", callbacks.Apply(1.5, lambda x: x * 2)) + +print("--- Counter.Check: a handle argument and a bool result") +print("check:", c.Check(lambda handle, n: callbacks.Counter(handle=handle).N == n)) + +print("--- a bound method") + +class Box(object): + def __init__(self): + self.items = [] + + def add(self, i, label): + self.items.append((i, label)) + +box = Box() +callbacks.Each(2, box.add) +print("box:", box.items) + +print("--- called from another goroutine") +callbacks.InGoroutine(lambda i: print("goroutine:", i)) + +print("--- an exception in a callback is reported, and Go carries on") +seen = [] + +def boom(i, label): + seen.append(i) + raise ValueError("boom %d" % i) + +stderr, sys.stderr = sys.stderr, io.StringIO() +try: + callbacks.Each(3, boom) + reported = sys.stderr.getvalue() +finally: + sys.stderr = stderr +print("calls:", len(seen), "reported:", reported.count("ValueError: boom")) + +print("--- a callback Go keeps and calls after the call it was passed to returned") +# Run in a child process: a callback that outlives its call must not crash it +# (cffi, pybind11 and nanobind refuse the call, cffi with a warning on stderr, +# which the output compared here mustn't depend on). +here = os.path.dirname(os.path.abspath(__file__)) +child = subprocess.run( + [sys.executable, "-c", "import callbacks\ndef kept(i): pass\ncallbacks.Keep(kept)\ncallbacks.CallKept(1)\n"], + cwd=here, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, +) +print("kept: exit code", child.returncode) +if child.returncode != 0: + print(child.stderr.decode(errors="replace")) + +print("OK") diff --git a/_examples/pyerrors/test.py b/_examples/pyerrors/test.py index c729b118..de1a67f2 100644 --- a/_examples/pyerrors/test.py +++ b/_examples/pyerrors/test.py @@ -5,6 +5,8 @@ ## py2/py3 compat from __future__ import print_function +import threading + import pyerrors def div(a, b): @@ -30,3 +32,40 @@ def new_mystring(s): new_mystring("hello") print("OK") + + +# Concurrent calls must not swap or drop errors between Python threads: each +# thread's exception (or lack of one) must match what it, specifically, did. +def race(): + n = 200 + bad = [] + lock = threading.Lock() + + def worker(i): + try: + if i % 2 == 0: + pyerrors.Div(10, 0) + with lock: + bad.append((i, "missing exception")) + else: + r = pyerrors.Div(10, 1) + if r != 10: + with lock: + bad.append((i, "wrong result: %r" % r)) + except Exception as e: + if i % 2 == 1: + with lock: + bad.append((i, "unexpected exception: %s" % e)) + elif str(e) != "Divide by zero.": + with lock: + bad.append((i, "wrong message: %s" % e)) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not bad, bad + + +race() diff --git a/_examples/slices/slices.go b/_examples/slices/slices.go index baa5d7e6..82d6777b 100644 --- a/_examples/slices/slices.go +++ b/_examples/slices/slices.go @@ -32,6 +32,7 @@ type SliceInt32 []int32 type SliceInt64 []int64 type SliceComplex []complex128 +type SliceComplex64 []complex64 type SliceIface []interface{} @@ -61,6 +62,10 @@ func CmplxSqrt(arr SliceComplex) SliceComplex { return res } +func CmplxArray() [3]complex128 { + return [3]complex128{1 + 1i, 2 + 2i, 3 + 3i} +} + func GetEmptyMatrix(xSize int, ySize int) [][]bool { result := [][]bool{} diff --git a/_examples/slices/test.py b/_examples/slices/test.py index 143f4533..b503909a 100644 --- a/_examples/slices/test.py +++ b/_examples/slices/test.py @@ -44,6 +44,19 @@ assert math.isclose(root_squared.real, orig.real) assert math.isclose(root_squared.imag, orig.imag) +# complex elements: assignment and append, in both float widths, and reading an array +cmplx[0] = 3 + 4j +assert cmplx[0] == 3 + 4j +cmplx.append(1 - 2j) +assert len(cmplx) == 17 and cmplx[16] == 1 - 2j + +cmplx64 = slices.SliceComplex64([1 + 2j, 3.5 - 4.25j]) +cmplx64[1] = -0.5 + 8j +cmplx64.append(2j) +assert list(cmplx64) == [1 + 2j, -0.5 + 8j, 2j] + +cmplx_arr = slices.CmplxArray() +assert len(cmplx_arr) == 3 and cmplx_arr[2] == 3 + 3j matrix = slices.GetEmptyMatrix(4,4) for i in range(4): diff --git a/bind/backend.go b/bind/backend.go new file mode 100644 index 00000000..8eb09977 --- /dev/null +++ b/bind/backend.go @@ -0,0 +1,56 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "fmt" + "os" + "strings" +) + +// BackendEnvVar is the environment variable that selects which tool is used +// to bind the generated cgo shim to CPython. +const BackendEnvVar = "GOPY_BACKEND" + +// Backend names a CPython binding tool. +type Backend string + +const ( + BackendPyBindGen Backend = "pybindgen" // default + BackendCFFI Backend = "cffi" + BackendPyBind11 Backend = "pybind11" + BackendNanobind Backend = "nanobind" + BackendCAPI Backend = "capi" +) + +// backends lists every known backend. +var backends = []Backend{ + BackendPyBindGen, + BackendCFFI, + BackendPyBind11, + BackendNanobind, + BackendCAPI, +} + +// BackendFromEnv returns the backend selected by GOPY_BACKEND. +// An unset or empty variable selects pybindgen. +func BackendFromEnv() (Backend, error) { + return parseBackend(os.Getenv(BackendEnvVar)) +} + +func parseBackend(v string) (Backend, error) { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "" { + return BackendPyBindGen, nil + } + names := make([]string, len(backends)) + for i, b := range backends { + names[i] = string(b) + if string(b) == v { + return b, nil + } + } + return "", fmt.Errorf("gopy: unknown %s=%q (valid values: %s)", BackendEnvVar, v, strings.Join(names, ", ")) +} diff --git a/bind/backend_test.go b/bind/backend_test.go new file mode 100644 index 00000000..537bbbaa --- /dev/null +++ b/bind/backend_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "strings" + "testing" +) + +func TestParseBackend(t *testing.T) { + for _, tc := range []struct { + in string + want Backend + errPart string + }{ + {in: "", want: BackendPyBindGen}, + {in: "pybindgen", want: BackendPyBindGen}, + {in: " PyBindGen ", want: BackendPyBindGen}, + {in: "cffi", want: BackendCFFI}, + {in: "pybind11", want: BackendPyBind11}, + {in: "nanobind", want: BackendNanobind}, + {in: "capi", want: BackendCAPI}, + {in: "cgo", errPart: "unknown GOPY_BACKEND"}, + {in: "bogus", errPart: "unknown GOPY_BACKEND"}, + } { + got, err := parseBackend(tc.in) + if tc.errPart != "" { + if err == nil || !strings.Contains(err.Error(), tc.errPart) { + t.Errorf("parseBackend(%q): got err=%v, want error containing %q", tc.in, err, tc.errPart) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("parseBackend(%q) = %q, %v; want %q", tc.in, got, err, tc.want) + } + } +} + +func TestBackendFromEnv(t *testing.T) { + t.Setenv(BackendEnvVar, "") + if got, err := BackendFromEnv(); err != nil || got != BackendPyBindGen { + t.Errorf("unset: got %q, %v", got, err) + } + t.Setenv(BackendEnvVar, "bogus") + if _, err := BackendFromEnv(); err == nil { + t.Error("bogus value: want error") + } +} diff --git a/bind/bind.go b/bind/bind.go index 93e78af8..f60acf66 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -31,6 +31,8 @@ type BindCfg struct { // gopy version string embedded in this binary, stamped into generated // file headers so output can be traced back to the release that produced it Version string + // tool used to bind the cgo shim to CPython, see BackendFromEnv + Backend Backend } // ErrorList is a list of errors diff --git a/bind/capi.go b/bind/capi.go new file mode 100644 index 00000000..1e3d9546 --- /dev/null +++ b/bind/capi.go @@ -0,0 +1,22 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" +) + +// The capi backend (GOPY_BACKEND=capi) is the default backend without +// pybindgen: the same cgo shim, which calls the CPython C API itself, and the +// same build (the generated .c compiled into the Go shared library). +// Only build.py differs: capi_build.py records the pybindgen calls gopy +// writes, and writes .c from them itself. + +//go:embed capi_build.py +var capiBuildPy string + +func (g *pyGen) isCAPI() bool { + return g.cfg.Backend == BackendCAPI +} diff --git a/bind/capi_build.py b/bind/capi_build.py new file mode 100644 index 00000000..b01c4bea --- /dev/null +++ b/bind/capi_build.py @@ -0,0 +1,211 @@ +# python build stubs for package @NAME@ (capi backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes +# @NAME@.c itself: the same CPython C-API wrappers pybindgen would write, for +# the same cgo shim, so nothing beyond the interpreter is needed to build. +# Every conversion below follows pybindgen's own generated code, so that a +# wrongly-typed argument raises the same exception under either backend. + + +class Retval(object): + def __init__(self, ctype, caller_owns_return=False, *a, **kw): + self.ctype = ctype + + +class Param(object): + def __init__(self, ctype, name, transfer_ownership=True, *a, **kw): + self.ctype = ctype + self.name = name + + +retval = Retval +param = Param + + +class Function(object): + def __init__(self, name, ret, params, checked=False, frees_string=False): + self.name = name + self.ret = ret.ctype if ret is not None else None + self.params = params + self.checked = checked + self.frees_string = frees_string + + +class Module(object): + def __init__(self, name, *a, **kw): + self.name = name # the compiled extension's import name, e.g. "_hi" + self.includes = [] + self.funcs = [] + + def add_include(self, inc): + self.includes.append(inc) + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append(Function(name, ret, params)) + + def generate(self, out): + out.write(MODULE_HEAD.replace("@INCLUDES@", "\n".join("#include " + i for i in self.includes))) + for fn in self.funcs: + out.write(wrapper(self.name, fn)) + out.write("static PyMethodDef %s_functions[] = {\n" % self.name) + for fn in self.funcs: + flags = "METH_VARARGS|METH_KEYWORDS" if fn.params else "METH_NOARGS" + out.write(' {"%s", (PyCFunction)%s, %s, "%s"},\n' % (fn.name, wrap_name(self.name, fn), flags, doc(fn))) + out.write(" {NULL, NULL, 0, NULL}\n};\n") + out.write(MODULE_TAIL.replace("@MOD@", self.name)) + out.close() + + +# failure_expression is never anything but '' from gopy, so a checked +# function's only check is PyErr_Occurred(), as with pybindgen. +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.funcs.append(Function(name, retval, params, checked=True)) + + +# As above, and the char* the function returns is the caller's to free. +def add_checked_string_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.funcs.append(Function(name, retval, params, checked=True, frees_string=True)) + + +# ctype -> (PyArg_Parse format, C type parsed into, maximum value or None). +# Narrow integers parse as int, and only their maximum is checked. +PARSE = { + "int64_t": ("L", "int64_t", None), + "uint64_t": ("K", "uint64_t", None), + "int": ("i", "int", None), + "int32_t": ("i", "int", None), + "uint32_t": ("I", "unsigned int", None), + "int16_t": ("i", "int", "0x7fff"), + "uint16_t": ("i", "int", "0xffff"), + "int8_t": ("i", "int", "0x7f"), + "uint8_t": ("i", "int", "0xff"), + "double": ("d", "double", None), + "float": ("f", "float", None), + "bool": ("O", "PyObject *", None), + "char*": ("s", "const char *", None), + "PyObject*": ("O", "PyObject *", None), +} + +# ctype -> Py_BuildValue arguments for the result in "retval". Py_BuildValue +# turns a NULL char* into None, and passes on a NULL PyObject* as an error. +BUILD = { + "int64_t": '"L", retval', + "uint64_t": '"K", retval', + "int": '"i", retval', + "int32_t": '"i", retval', + "int16_t": '"i", retval', + "uint16_t": '"i", retval', + "int8_t": '"i", retval', + "uint8_t": '"i", (int)retval', + "uint32_t": '"N", PyLong_FromUnsignedLong(retval)', + "double": '"d", retval', + "float": '"f", retval', + "bool": '"N", PyBool_FromLong(retval)', + "char*": '"s", retval', + "PyObject*": '"N", retval', +} + + +def wrap_name(mod, fn): + return "_wrap_%s_%s" % (mod, fn.name) + + +def doc(fn): + sig = "%s(%s)\\n\\n" % (fn.name, ", ".join(p.name for p in fn.params)) + return sig + "\\n".join("type: %s: %s" % (p.name, p.ctype.replace("*", " *")) for p in fn.params) + + +def wrapper(mod, fn): + for ctype in [p.ctype for p in fn.params] + ([fn.ret] if fn.ret else []): + if ctype not in PARSE: + raise ValueError("capi backend: unsupported C type %r in %s" % (ctype, fn.name)) + if not fn.params: + lines = ["static PyObject *\n%s(PyObject *self, PyObject *unused)\n{" % wrap_name(mod, fn)] + else: + lines = ["static PyObject *\n%s(PyObject *self, PyObject *args, PyObject *kwargs)\n{" % wrap_name(mod, fn)] + if fn.ret: + lines.append(" %s retval;" % fn.ret) + # locals are prefixed, so that no parameter name can collide with them + call_args = [] + for p in fn.params: + lines.append(" %s a_%s;" % (PARSE[p.ctype][1], p.name)) + if p.ctype == "bool": + call_args.append("(bool)PyObject_IsTrue(a_%s)" % p.name) + elif p.ctype == "char*": + call_args.append("(char *)a_" + p.name) + else: + call_args.append("a_" + p.name) + if fn.params: + lines.append( + " static const char *keywords[] = {%s, NULL};" % ", ".join('"%s"' % p.name for p in fn.params) + ) + lines.append( + ' if (!PyArg_ParseTupleAndKeywords(args, kwargs, "%s", (char **)keywords, %s)) {\n' + " return NULL;\n }" + % ("".join(PARSE[p.ctype][0] for p in fn.params), ", ".join("&a_" + p.name for p in fn.params)) + ) + for p in fn.params: + limit = PARSE[p.ctype][2] + if limit: + lines.append( + " if (a_%s > %s) {\n" + ' PyErr_SetString(PyExc_ValueError, "Out of range");\n' + " return NULL;\n }" % (p.name, limit) + ) + call = "%s(%s);" % (fn.name, ", ".join(call_args)) + lines.append(" " + ("retval = " + call if fn.ret else call)) + if fn.checked: + cleanup = " if (retval != NULL) free(retval);\n" if fn.frees_string else "" + lines.append(" if (PyErr_Occurred()) {\n%s return NULL;\n }" % cleanup) + if not fn.ret: + lines.append(" Py_RETURN_NONE;") + elif fn.frees_string: + lines.append(" PyObject *py_retval = Py_BuildValue(%s);" % BUILD[fn.ret]) + lines.append(" free(retval);") + lines.append(" return py_retval;") + else: + lines.append(" return Py_BuildValue(%s);" % BUILD[fn.ret]) + lines.append("}\n\n") + return "\n".join(lines) + + +MODULE_HEAD = """/* This file was generated by gopy's capi backend. Do not edit. */ +#define PY_SSIZE_T_CLEAN +#include +#include + +@INCLUDES@ + +""" + +# PyInit_ starts its own line: on Windows, cmd_build.go adds +# __declspec(dllexport) after any " PyInit_" in the file, which +# PyMODINIT_FUNC already has. +MODULE_TAIL = """ +static struct PyModuleDef @MOD@_moduledef = { + PyModuleDef_HEAD_INIT, + "@MOD@", + NULL, + -1, + @MOD@_functions, +}; + +PyMODINIT_FUNC +PyInit_@MOD@(void) +{ + return PyModule_Create(&@MOD@_moduledef); +} +""" + + +mod = Module('_@NAME@') +mod.add_include('"@NAME@_go.h"') +mod.add_function('GoPyInit', None, []) +mod.add_function('DecRef', None, [param('int64_t', 'handle')]) +mod.add_function('IncRef', None, [param('int64_t', 'handle')]) +mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) +mod.add_function('_gopy_clear_go_tls', None, []) diff --git a/bind/cffi.go b/bind/cffi.go new file mode 100644 index 00000000..3eb397f6 --- /dev/null +++ b/bind/cffi.go @@ -0,0 +1,217 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" +) + +// The cffi backend (GOPY_BACKEND=cffi) uses the shared no-API cgo shim (see +// noapi.go): a plain C shared library that Python loads with cffi. Errors +// are recorded for the Python side to raise, instead of being set with +// PyErr_SetString. + +//go:embed cffi_build.py +var cffiBuildPy string + +// same argument positions as goPreamble: 1 = name of package, 2 = cmdstr, +// 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 8 = C trampolines for +// callbacks (see cffi_callback.go), 10 = gopy version. +const goPreambleCFFI = `/* +cgo stubs for package %[1]s, for use with cffi or pybind11. +File is generated by gopy version %[10]s. Do not edit. +%[2]s +*/ + +package main + +/* +#include +#include +// pybind11's consumer #includes this same preamble (via the generated +// header) as C++, where bool/thread_local are keywords, not the C ones below. +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L)) +typedef uint8_t bool; +#endif +#if defined(__cplusplus) +#define GOPY_THREAD_LOCAL thread_local +#else +#define GOPY_THREAD_LOCAL _Thread_local +#endif +// Thread-local storage for the last error, read by the python side right +// after the call that set it, on the same OS thread. Windows gets its own +// branch: mingw's compiler-provided thread-local storage (_Thread_local) +// needs TLS support that a DLL loaded after process start (as cffi and +// ctypes do) isn't guaranteed to have, so this uses the OS API instead, +// which carries no such restriction (see TlsAlloc/TlsGetValue/TlsSetValue). +#ifdef _WIN32 +#include +static DWORD gopy_err_tls = TLS_OUT_OF_INDEXES; +static inline void gopy_err_tls_init(void) { gopy_err_tls = TlsAlloc(); } +static inline char* gopy_err_get(void) { return (char*)TlsGetValue(gopy_err_tls); } +static inline void gopy_err_put(char* msg) { TlsSetValue(gopy_err_tls, msg); } +#else +static inline void gopy_err_tls_init(void) {} +static GOPY_THREAD_LOCAL char* gopy_err_msg; +static inline char* gopy_err_get(void) { return gopy_err_msg; } +static inline void gopy_err_put(char* msg) { gopy_err_msg = msg; } +#endif +static inline void gopy_set_err(char* msg) { + free(gopy_err_get()); + gopy_err_put(msg); +} +static inline char* gopy_take_err(void) { + char* msg = gopy_err_get(); + gopy_err_put(NULL); + return msg; +} +%[8]s +*/ +import "C" +import ( + "runtime" + "sync" + "unsafe" + "github.com/go-python/gopy/gopyh" // handler + %[6]s +) + +func main() { + %[7]s +} + +//export GoPyInit +func GoPyInit() { + %[7]s +} + +// type for the handle -- int64 for speed (can switch to string) +type GoHandle %[4]s +type CGoHandle %[5]s + +// DecRef decrements the reference count for the specified handle +// and deletes it it goes to zero. +//export DecRef +func DecRef(handle CGoHandle) { + gopyh.DecRef(gopyh.CGoHandle(handle)) +} + +// IncRef increments the reference count for the specified handle. +//export IncRef +func IncRef(handle CGoHandle) { + gopyh.IncRef(gopyh.CGoHandle(handle)) +} + +// NumHandles returns the number of handles currently in use. +//export NumHandles +func NumHandles() int { + return gopyh.NumHandles() +} + +// RequestGC runs Go's garbage collector on a dedicated goroutine, and waits. +//export RequestGC +func RequestGC() { + done := make(chan struct{}) + _gcReq <- done + <-done +} + +var _gcReq = make(chan chan struct{}) + +func init() { + C.gopy_err_tls_init() + go func() { + for done := range _gcReq { + runtime.GC() + close(done) + } + }() +} + +// The error of the last call on the calling OS thread, for the python side +// to raise; gopy_err_msg is thread-local so concurrent calls on other +// threads don't see it. +func gopySetError(kind, msg string) { + C.gopy_set_err(C.CString(kind + ":" + msg)) +} + +// GopyTakeError returns "Kind:message" for the error recorded by the last +// call on the calling thread and clears it, or NULL if there is none. +// Free with GopyFreeString. +//export GopyTakeError +func GopyTakeError() *C.char { + return C.gopy_take_err() +} + +// GopyFreeString frees a string returned by this library. +//export GopyFreeString +func GopyFreeString(s *C.char) { + C.free(unsafe.Pointer(s)) +} + +// gopyCallbackScope guards a python callback passed to Go, which only exists +// while the python call it was passed to is running (see cffi_callback.go): +// close waits for callbacks that are running, and then refuses new ones. +type gopyCallbackScope struct { + mu sync.RWMutex + done bool +} + +func (s *gopyCallbackScope) enter() bool { + s.mu.RLock() + if s.done { + s.mu.RUnlock() + println("gopy: callback called after the python call it was passed to returned") + return false + } + return true +} + +func (s *gopyCallbackScope) leave() { s.mu.RUnlock() } + +func (s *gopyCallbackScope) close() { + s.mu.Lock() + s.done = true + s.mu.Unlock() +} + +// boolGoToPy converts a Go bool to python-compatible C.char +func boolGoToPy(b bool) C.char { + if b { + return 1 + } + return 0 +} + +// boolPyToGo converts a python-compatible C.Char to Go bool +func boolPyToGo(b C.char) bool { + return b != 0 +} + +// errorGoToPy converts a Go error to python-compatible C.CString +func errorGoToPy(e error) *C.char { + if e != nil { + return C.CString(e.Error()) + } + return C.CString("") +} + +// complex values cross as two floats, see isComplexShim in cffi.go +func complex64GoToPyCFFI(c complex64) (C.float, C.float) { + return C.float(real(c)), C.float(imag(c)) +} + +func complex64PyToGoCFFI(re, im C.float) complex64 { + return complex(float32(re), float32(im)) +} + +func complex128GoToPyCFFI(c complex128) (C.double, C.double) { + return C.double(real(c)), C.double(imag(c)) +} + +func complex128PyToGoCFFI(re, im C.double) complex128 { + return complex(float64(re), float64(im)) +} +` diff --git a/bind/cffi_build.py b/bind/cffi_build.py new file mode 100644 index 00000000..c4ff530b --- /dev/null +++ b/bind/cffi_build.py @@ -0,0 +1,276 @@ +# python build stubs for package @NAME@ (cffi backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes a cffi +# (ABI mode) module, _@NAME@.py, that loads @NAME@_go@LIBEXT@ directly. The +# exact C types come from the extern declarations in cgo's @NAME@_go.h. + +import os +import re +import sys + +import cffi + + +def retval(ctype, *a, **kw): + return ctype + + +def param(ctype, name, *a, **kw): + return (ctype, name) + + +class Module(object): + def __init__(self, name): + self.name = name + self.header = None + self.funcs = [] + + def add_include(self, inc): + self.header = inc.strip('"') + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append((name, ret, params)) + + def generate(self): + here = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(here, self.header)) as f: + typedefs, cdefs = go_decls(f.read()) + out = [MODULE_HEAD.replace("@CDEFS@", repr("\n".join(typedefs + list(cdefs.values()))))] + for name, ret, params in self.funcs: + out.append(wrapper(name, ret, params, name in cdefs)) + # Slice_byte's converters exchange a raw pointer+length instead of a + # PyObject* (see gen_slice.go); they are recognized by name here + # rather than recorded via add_function, since their python bodies + # aren't derived from a plain C signature. + if "Slice_byte_from_bytes" in cdefs and "Slice_byte_to_bytes_ptr" in cdefs: + out.append(BYTES_FUNCS) + with open(os.path.join(here, self.name + ".py"), "w") as f: + f.write("\n".join(out)) + + +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.add_function(name, retval, params) + + +add_checked_string_function = add_checked_function + + +def go_decls(header): + """Returns the Go typedefs (plus any "struct X { ... };" body -- cgo emits + one ahead of an export that returns two values, which is how a complex + number is returned, see isCFFIComplex in cffi.go), and {function name: + cdef line} for the functions cgo exports. + """ + typedefs = [] + decls = {} + ffi = cffi.FFI() + lines = header.split("\n") + i = 0 + while i < len(lines): + line = lines[i] + m = re.match(r"struct (\w+) \{$", line) + if m: + block = [line] + i += 1 + while i < len(lines) and lines[i] != "};": + block.append(lines[i]) + i += 1 + block.append("};") + i += 1 + decl = "\n".join(block) + try: + ffi.cdef(decl) + typedefs.append(decl) + except Exception as err: + print("gopy: cffi cannot declare struct %s: %s" % (m.group(1), err), file=sys.stderr) + continue + if re.match(r"typedef [\w ]+ Go\w+;$", line): + try: + ffi.cdef(line) + typedefs.append(line) + except Exception: + pass # e.g. GoComplex64: not used by exports + i += 1 + continue + m = re.match(r"extern (.*?(\w+)\(.*\));$", line.replace("__declspec(dllexport) ", "")) + if not m or "_GoString_" in line: + i += 1 + continue + # cffi takes a plain char as a byte string only; the integer kinds + # (bool, int8, byte) are passed as ints, with the same C ABI. + decl = re.sub(r"\b(?()", see cffi_callback.go). + The wrapper passes it on to Go as _cb_, which is kept referenced by + this local variable until the Go call returns: cffi frees a callback as + soon as nothing refers to it. + + If the callable raises, cffi prints the traceback and returns 0 to Go. + """ + ret, _, rest = ctype[len("callback:"):].partition("(") + ctypes = [t for t in rest[:-1].split(",") if t] + names = ["a%d" % i for i in range(len(ctypes))] + conv = [] + for n, t in zip(names, ctypes): + if t == "char*": + conv.append('_ffi.string(%s).decode("utf-8")' % n) + elif t == "bool": + conv.append("bool(%s)" % n) + else: + conv.append(n) + call = "%s(%s)" % (pname, ", ".join(conv)) + if ret == "void": + body = call + elif ret == "bool": + body = "return 1 if %s else 0" % call + else: + body = "return %s" % call + + def cdecl_type(t): + return "unsigned char" if t == "bool" else t + + cdecl = "%s(%s)" % (cdecl_type(ret), ", ".join(cdecl_type(t) for t in ctypes)) + return "\n".join([ + " if not callable(%s):" % pname, + " raise TypeError('argument %d must be callable, not %%s' %% type(%s).__name__)" % (argn, pname), + " def _cbfn_%s(%s):" % (pname, ", ".join(names)), + " %s" % body, + " _cb_%s = _ffi.callback(%r, _cbfn_%s)" % (pname, cdecl, pname), + ]) + + +MODULE_HEAD = '''# python bindings for package @NAME@ using cffi. +# File is generated by gopy version @VERSION@. Do not edit. +import builtins +import ctypes +import os +from operator import index as _index + +import cffi + +_ffi = cffi.FFI() +_ffi.cdef(@CDEFS@) +_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "@NAME@_go@LIBEXT@") +_lib = _ffi.dlopen(_path) +# Go cannot unload its runtime. cffi unloads _lib at interpreter shutdown but +# ctypes never unloads, so this second handle keeps the library loaded until +# the process exits, as for an extension module. +_pin = ctypes.CDLL(_path) + + +def _enc(s, argn): + if not isinstance(s, str): + raise TypeError("argument %d must be str, not %s" % (argn, type(s).__name__)) + return s.encode("utf-8") + + +def _dec(p): + """Copies a malloc'd C string into a str and frees it.""" + if p == _ffi.NULL: + return None + try: + return _ffi.string(p).decode("utf-8") + finally: + _lib.GopyFreeString(p) + + +def _s8(v): + return ((v + 128) & 0xFF) - 128 + + +def _check(): + """Raises the exception, if any, that the last Go call recorded.""" + e = _lib.GopyTakeError() + if e != _ffi.NULL: + kind, _, msg = _dec(e).partition(":") + raise getattr(builtins, kind, RuntimeError)(msg) + +''' + +BYTES_FUNCS = ''' +def Slice_byte_from_bytes(b): + if not isinstance(b, (bytes, bytearray)): + raise TypeError("argument 1 must be bytes, not %s" % type(b).__name__) + _r = _lib.Slice_byte_from_bytes(_ffi.from_buffer(b), len(b)) + _check() + return _r + + +def Slice_byte_to_bytes(handle): + n = _lib.Slice_byte_to_bytes_len(handle) + _check() + if n == 0: + return b"" + ptr = _lib.Slice_byte_to_bytes_ptr(handle) + _check() + try: + return bytes(_ffi.buffer(ptr, n)) + finally: + _lib.Slice_byte_free_ptr(ptr) +''' + +mod = Module('_@NAME@') +mod.add_include('"@NAME@_go.h"') +mod.add_function('GoPyInit', None, []) +mod.add_function('DecRef', None, [param('int64_t', 'handle')]) +mod.add_function('IncRef', None, [param('int64_t', 'handle')]) +mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) +mod.add_function('_gopy_clear_go_tls', None, []) diff --git a/bind/cffi_callback.go b/bind/cffi_callback.go new file mode 100644 index 00000000..284b6b11 --- /dev/null +++ b/bind/cffi_callback.go @@ -0,0 +1,245 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "bytes" + "fmt" + "go/types" + "strings" +) + +// A Python callable passed to Go as a func-typed argument crosses the cffi +// boundary as a C function pointer: the python side wraps the callable in an +// ffi.callback (see callback_setup in cffi_build.py), and the Go closure +// built here calls that pointer through a small static C trampoline, since +// cgo cannot call a C function pointer directly. +// +// The callback only lives as long as the Go call it was passed to, so the +// closure goes through a gopyCallbackScope that is closed when that call +// returns, instead of jumping to freed memory if Go kept it any longer. +// +// Supported: parameters that are pointer/interface handles, numbers, bool, +// string or interface{} (which arrives as a string, as with pybindgen), and a +// result that is a number or bool. Any other callback type makes the +// function be skipped, see genFuncSig. + +// cffiTrampolinesKey stands in for the trampolines in the cgo preamble, +// which is written before the callback types that need them are known. +const cffiTrampolinesKey = "@@GOPY_CFFI_TRAMPOLINES@@" + +// cffiCBParam is one parameter of a callback. +type cffiCBParam struct { + name string // name of the parameter in the func literal + gotyp string // its Go type + ctype string // how it crosses: int64_t, uint64_t, double, char* or bool + pre string // Go statements to run before the call, if any + conv string // Go expression converting it to ctype +} + +// cffiCBResult is the result of a callback. +type cffiCBResult struct { + gotyp string // its Go type + ctype string // how it crosses: int64_t, uint64_t, double or bool +} + +type cffiCallback struct { + params []cffiCBParam + ret *cffiCBResult // nil if the callback has no result +} + +// cffiBasicCType returns how a value of a basic Go type crosses, or "" if it +// can't. +func cffiBasicCType(k types.BasicKind) string { + switch { + case types.Int <= k && k <= types.Int64: + return "int64_t" + case types.Uint <= k && k <= types.Uintptr: + return "uint64_t" + case k == types.Float32 || k == types.Float64: + return "double" + case k == types.Bool: + return "bool" + case k == types.String: + return "char*" + } + return "" +} + +// cffiCType returns the C type used for ctype in C code: bool is a byte. +func cffiCType(ctype string) string { + if ctype == "bool" { + return "uint8_t" + } + return ctype +} + +// cffiCallback returns how to pass sym, a func-typed argument, to Go +// as a callback, or nil if its type isn't supported. +func (g *pyGen) cffiCallback(sym *symbol) *cffiCallback { + sig, ok := sym.GoType().Underlying().(*types.Signature) + if !ok || sig.Results().Len() > 1 || sig.Variadic() { + return nil + } + cb := &cffiCallback{} + for i := 0; i < sig.Params().Len(); i++ { + p, ok := cffiCallbackParam(sig.Params().At(i), i) + if !ok { + return nil + } + cb.params = append(cb.params, p) + } + if sig.Results().Len() == 1 { + r, ok := cffiCallbackResult(sig.Results().At(0).Type()) + if !ok { + return nil + } + cb.ret = r + } + return cb +} + +// cStringPre returns the Go statements that make the C string _c from +// expr, for the duration of the call. The python side copies it. +func cStringPre(nm, expr string) string { + return fmt.Sprintf("_c%[1]s := %[2]s\ndefer C.free(unsafe.Pointer(_c%[1]s))\n", nm, expr) +} + +func cffiCallbackParam(v *types.Var, i int) (cffiCBParam, bool) { + typ := v.Type() + vsym := current.symtype(typ) + if vsym == nil { + return cffiCBParam{}, false + } + nm := pySafeArg(v.Name(), i) + p := cffiCBParam{name: nm, gotyp: current.typeGoName(typ)} + + if vsym.hasHandle() && vsym.isPtrOrIface() { + p.ctype = "int64_t" + p.conv = fmt.Sprintf("C.int64_t(%s(%s)%s)", vsym.go2py, nm, vsym.go2pyParenEx) + return p, true + } + if vsym.goname == "interface{}" { + p.ctype, p.conv = "char*", "_c"+nm + p.pre = cStringPre(nm, fmt.Sprintf("%s(%s)%s", vsym.go2py, nm, vsym.go2pyParenEx)) + return p, true + } + bt, ok := typ.Underlying().(*types.Basic) + if !ok { + return p, false + } + switch p.ctype = cffiBasicCType(bt.Kind()); p.ctype { + case "": + return p, false + case "bool": + p.conv = fmt.Sprintf("C.uint8_t(boolGoToPy(bool(%s)))", nm) + case "char*": + p.conv = "_c" + nm + p.pre = cStringPre(nm, fmt.Sprintf("C.CString(string(%s))", nm)) + default: + p.conv = fmt.Sprintf("C.%s(%s)", p.ctype, nm) + } + return p, true +} + +func cffiCallbackResult(typ types.Type) (*cffiCBResult, bool) { + bt, ok := typ.Underlying().(*types.Basic) + if !ok { + return nil, false + } + switch ctype := cffiBasicCType(bt.Kind()); ctype { + case "", "char*": // the python side has no way to give Go a string it owns + return nil, false + default: + return &cffiCBResult{gotyp: current.typeGoName(typ), ctype: ctype}, true + } +} + +// pyType is the type given to the callback's parameter in build.py, which +// callback_setup in cffi_build.py takes apart: +// callback:() +func (cb *cffiCallback) pyType() string { + ret := "void" + if cb.ret != nil { + ret = cb.ret.ctype + } + ts := make([]string, len(cb.params)) + for i, p := range cb.params { + ts[i] = p.ctype + } + return "callback:" + ret + "(" + strings.Join(ts, ",") + ")" +} + +// cffiCallbackPrologue returns the Go statements that set up the callback +// argument named anm, ahead of cffiCallbackLit. +func cffiCallbackPrologue(anm string) string { + return fmt.Sprintf("_cbfp_%[1]s := %[1]s\n_cbs_%[1]s := new(gopyCallbackScope)\ndefer _cbs_%[1]s.close()\n", anm) +} + +// cffiCallbackLit returns a Go func literal that calls the Python callable +// passed as the argument named anm. +func (g *pyGen) cffiCallbackLit(cb *cffiCallback, anm string) string { + var decl, pre []string + args := []string{"_cbfp_" + anm} + for _, p := range cb.params { + decl = append(decl, p.name+" "+p.gotyp) + pre = append(pre, p.pre) + args = append(args, p.conv) + } + call := fmt.Sprintf("C.gopy_cb_%d(%s)", g.cffiTrampoline(cb), strings.Join(args, ", ")) + result := "" + if cb.ret != nil { + // named, so that a refused call returns its zero value + result = " (_r " + cb.ret.gotyp + ")" + if cb.ret.ctype == "bool" { + call += " != 0" + } + call = "return " + cb.ret.gotyp + "(" + call + ")" + } + return fmt.Sprintf("func(%[1]s)%[2]s {\nif !_cbs_%[3]s.enter() {\nreturn\n}\ndefer _cbs_%[3]s.leave()\n%[4]s%[5]s\n}", + strings.Join(decl, ", "), result, anm, strings.Join(pre, ""), call) +} + +// cffiTrampoline returns the number of the C trampoline that calls a callback +// like cb, adding it if it is the first. +func (g *pyGen) cffiTrampoline(cb *cffiCallback) int { + for i, c := range g.cbs { + if c.pyType() == cb.pyType() { + return i + } + } + g.cbs = append(g.cbs, cb) + return len(g.cbs) - 1 +} + +// spliceCFFITrampolines writes the trampolines into the cgo preamble. +func (g *pyGen) spliceCFFITrampolines() { + if !g.isCFFI() { + return + } + var c strings.Builder + for i, cb := range g.cbs { + params := []string{"void* f"} + ptypes := []string{} + args := []string{} + for j, p := range cb.params { + t := cffiCType(p.ctype) + params = append(params, fmt.Sprintf("%s a%d", t, j)) + ptypes = append(ptypes, t) + args = append(args, fmt.Sprintf("a%d", j)) + } + if len(ptypes) == 0 { + ptypes = append(ptypes, "void") + } + ret, retStmt := "void", "" + if cb.ret != nil { + ret, retStmt = cffiCType(cb.ret.ctype), "return " + } + fmt.Fprintf(&c, "static inline %s gopy_cb_%d(%s) { %s((%s (*)(%s))f)(%s); }\n", + ret, i, strings.Join(params, ", "), retStmt, ret, strings.Join(ptypes, ", "), strings.Join(args, ", ")) + } + b := bytes.Replace(g.gofile.buf.Bytes(), []byte(cffiTrampolinesKey), []byte(c.String()), 1) + g.gofile.buf = bytes.NewBuffer(b) +} diff --git a/bind/cxx_args.inc b/bind/cxx_args.inc new file mode 100644 index 00000000..a8c7cb8b --- /dev/null +++ b/bind/cxx_args.inc @@ -0,0 +1,120 @@ +// Argument conversion shared by the pybind11 and nanobind backends, spliced +// into each one's generated .cpp in place of @ARG_HELPERS@ (see +// cxxArgHelpers in pybind11.go). Each binding takes its arguments as raw +// python objects and converts them here, the same way pybindgen's generated +// code does (PyArg_ParseTuple, with the same format code per C type, plus +// pybindgen's own range checks), so that a wrongly-typed argument raises the +// same exception, with the same message, under every backend -- rather than +// pybind11's or nanobind's own "incompatible function arguments" TypeError. +// argn is the argument's 1-based position, as PyArg_ParseTuple counts it. +// +// gopy_raise must be defined first: it throws the C++ exception that hands +// the python error already set back to python (pybind11 and nanobind each +// spell that differently). Note that this text is spliced into a python +// string literal in build.py, so it must not contain any backslashes. + +// the type name PyArg_ParseTuple reports for o +static const char* gopy_arg_type(PyObject* o) { + return o == Py_None ? "None" : Py_TYPE(o)->tp_name; +} + +static void gopy_arg_error(PyObject* exc, const char* msg) { + PyErr_SetString(exc, msg); + gopy_raise(); +} + +// PyArg_ParseTuple's "s" +static const char* gopy_arg_str(PyObject* o, int argn) { + if (!PyUnicode_Check(o)) { + PyErr_Format(PyExc_TypeError, "argument %d must be str, not %.50s", argn, gopy_arg_type(o)); + gopy_raise(); + } + Py_ssize_t n; + const char* s = PyUnicode_AsUTF8AndSize(o, &n); + if (!s) { + gopy_raise(); + } + if ((Py_ssize_t)strlen(s) != n) { + gopy_arg_error(PyExc_ValueError, "embedded null character"); + } + return s; // borrowed from o, which outlives the call it is passed to +} + +// PyArg_ParseTuple's "L" +static long long gopy_arg_L(PyObject* o) { + long long v = PyLong_AsLongLong(o); + if (v == -1 && PyErr_Occurred()) { + gopy_raise(); + } + return v; +} + +// PyArg_ParseTuple's "K" +static unsigned long long gopy_arg_K(PyObject* o, int argn) { + if (!PyLong_Check(o)) { + PyErr_Format(PyExc_TypeError, "argument %d must be int, not %.50s", argn, gopy_arg_type(o)); + gopy_raise(); + } + return PyLong_AsUnsignedLongLongMask(o); +} + +// PyArg_ParseTuple's "i" +static int gopy_arg_i(PyObject* o) { + long v = PyLong_AsLong(o); + if (v == -1 && PyErr_Occurred()) { + gopy_raise(); + } + if (v > INT_MAX) { + gopy_arg_error(PyExc_OverflowError, "signed integer is greater than maximum"); + } + if (v < INT_MIN) { + gopy_arg_error(PyExc_OverflowError, "signed integer is less than minimum"); + } + return (int)v; +} + +// PyArg_ParseTuple's "i", then pybindgen's own check, for the C types it +// parses that way despite being narrower than int (it checks only the upper +// bound). +static int gopy_arg_i_max(PyObject* o, int max) { + int v = gopy_arg_i(o); + if (v > max) { + gopy_arg_error(PyExc_ValueError, "Out of range"); + } + return v; +} + +// PyArg_ParseTuple's "I" +static unsigned int gopy_arg_I(PyObject* o) { + unsigned long v = PyLong_AsUnsignedLongMask(o); + if (v == (unsigned long)-1 && PyErr_Occurred()) { + gopy_raise(); + } + return (unsigned int)v; +} + +// PyArg_ParseTuple's "d" (and, narrowed, "f") +static double gopy_arg_d(PyObject* o) { + double v = PyFloat_AsDouble(o); + if (v == -1.0 && PyErr_Occurred()) { + gopy_raise(); + } + return v; +} + +// pybindgen's bool: PyArg_ParseTuple's "O", then PyObject_IsTrue +static char gopy_arg_bool(PyObject* o) { + int v = PyObject_IsTrue(o); + if (v < 0) { + gopy_raise(); + } + return (char)v; +} + +// as the cffi backend checks a callback argument (pybindgen can't take one) +static void gopy_arg_callable(PyObject* o, int argn) { + if (!PyCallable_Check(o)) { + PyErr_Format(PyExc_TypeError, "argument %d must be callable, not %.50s", argn, Py_TYPE(o)->tp_name); + gopy_raise(); + } +} diff --git a/bind/cxxbuild.go b/bind/cxxbuild.go new file mode 100644 index 00000000..f832c984 --- /dev/null +++ b/bind/cxxbuild.go @@ -0,0 +1,214 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// How the C++ backends (pybind11, nanobind) compile and link their module, +// shared by gopy build (buildCXXModule in cmd_build.go, which runs these +// steps itself) and the Makefile written by gopy gen/pkg (genMakefileCXX). +// +// The Go side builds as a static archive (-buildmode=c-archive), not a +// shared library: the generated .cpp defines the callback trampolines Go +// calls into (see pybind11_callback.go) as well as calling into Go itself, +// and two separately-built shared libraries can't have a dependency cycle +// like that -- neither can exist as a complete, loadable file before the +// other -- so both sides' symbols are resolved in the single final link +// that CXXArgs describes. + +// NanobindCXXFlags are the flags nanobind's own build uses for libnanobind +// (see the comment at the top of its nb_combined.cpp), which the nanobind +// backend compiles into every module; harmless for the generated .cpp too. +var NanobindCXXFlags = []string{"-DNDEBUG", "-DNB_COMPACT_ASSERTIONS", "-fno-strict-aliasing"} + +// CXX returns the C++ compiler to use: $CXX, or else c++. +func CXX() string { + if cxx := os.Getenv("CXX"); cxx != "" { + return cxx + } + return "c++" +} + +// CXXArchive returns the file name of the static archive the cgo shim for +// package name builds as. +func CXXArchive(name string) string { + return name + "_go.a" +} + +// ExtModuleName returns the file name of the extension module _, +// preferring the interpreter's own suffix (e.g. .cpython-312-x86_64-linux-gnu.so) +// over libext. +func ExtModuleName(name, libext string, pycfg PyConfig) string { + if pycfg.ExtSuffix != "" { + return "_" + name + pycfg.ExtSuffix + } + return "_" + name + libext +} + +// CXXArgs returns the C++ compiler arguments that compile .cpp and +// srcs (the C++ library's own sources, if any) with libflags (its include +// directories and flags of its own), and link them with the archive into +// modlib. +func CXXArgs(name, modlib string, pycfg PyConfig, libflags, srcs []string) []string { + archive := CXXArchive(name) + // pycfg.CFlags/LdFlags quote each path (for the shell that CGO_CFLAGS/ + // CGO_LDFLAGS normally go through); these arguments go to the compiler + // directly, so unquote each field here. + unquote := func(fields []string) []string { + o := make([]string, len(fields)) + for i, f := range fields { + o[i] = strings.Trim(f, `"`) + } + return o + } + // modlib depends on libpython (wherever this VM's own one lives, e.g. + // not on the loader's default search path for a uv- or pyenv-managed + // Python); without an rpath, the loader only finds it if it happens to + // already be on its search path. + var libdir string + if m := regexp.MustCompile(`-L(\S+)`).FindStringSubmatch(pycfg.LdFlags); m != nil { + libdir = strings.Trim(m[1], `"`) + } + // The archive's Go runtime code calls into gopy_cb_N (defined in the + // .cpp), so the linker must be told to keep every object in it -- left + // to its own judgement, it would see nothing in the .cpp calling into + // the archive first and drop it as unused. GNU ld (Linux, and Windows' + // MinGW) and ld64 (macOS) spell that differently. + var archiveArgs []string + if runtime.GOOS == "darwin" { + archiveArgs = []string{"-Wl,-force_load," + archive} + } else { + archiveArgs = []string{"-Wl,--whole-archive", archive, "-Wl,--no-whole-archive"} + } + args := []string{"-std=c++17", "-fPIC", "-shared", "-O2"} + switch runtime.GOOS { + case "darwin": + args = append(args, "-Wl,-rpath,@loader_path") + if libdir != "" { + args = append(args, "-Wl,-rpath,"+libdir) + } + case "windows": + // No rpath equivalent; modlib depends on nothing but libpython, the + // Go side being a static archive rather than a separate DLL of its + // own. MinGW's own runtime (libstdc++/libgcc/libwinpthread), which + // g++ links dynamically by default, has no such fix available -- it + // isn't found by name alone unless its directory happens to be on + // PATH -- so link it in statically instead. The C runtime (ucrt) + // stays dynamic, shared with Python's own. + args = append(args, "-static-libgcc", "-static-libstdc++", + "-Wl,-Bstatic,--whole-archive", "-lwinpthread", "-Wl,--no-whole-archive", "-Wl,-Bdynamic") + default: + args = append(args, "-Wl,-rpath,$ORIGIN") + if libdir != "" { + args = append(args, "-Wl,-rpath,"+libdir) + } + } + args = append(args, libflags...) + args = append(args, unquote(strings.Fields(pycfg.CFlags))...) + args = append(args, name+".cpp") + args = append(args, srcs...) + args = append(args, archiveArgs...) + args = append(args, unquote(strings.Fields(pycfg.LdFlags))...) + // c-archive mode (unlike c-shared) doesn't resolve the Go runtime's own + // dependencies on these itself; TODO: verified only on Linux -- unclear + // yet whether Windows/macOS need anything of their own added here too. + if runtime.GOOS != "windows" { + args = append(args, "-lpthread", "-ldl", "-lm") + } + return append(args, "-o", modlib) +} + +// makeShellArg returns arg written into a Makefile recipe line, so that the +// shell make runs it with receives arg itself: make variable references +// ("$(...)") are left for make to expand, any other "$" is escaped from +// make, and anything the shell would split or expand is single-quoted. +// Each "'" inside closes the quoting, adds a double-quoted "'", and reopens +// it, rather than using a backslash: the double quote makes make hand the +// line to the shell, where Windows make's own argument splitting would +// otherwise drop the "'". +func makeShellArg(arg string) string { + if strings.HasPrefix(arg, "$(") { + return arg + } + arg = strings.ReplaceAll(arg, "$", "$$") + if arg == "" || strings.ContainsAny(arg, " \t\n'\"\\`*?[#~&;|<>()$") { + return "'" + strings.ReplaceAll(arg, "'", `'"'"'`) + "'" + } + return arg +} + +// MakefileTemplateCXX is the Makefile for the C++ backends: 1 = package +// name, 2 = gopy command, 3 = gencmd, 4 = vm, 5 = C++ compiler, 6 = the +// backend's own make variables (CXXLIBFOUND, CXXLIBFLAGS, CXXLIBSRCS), 7 = C++ compiler +// arguments, 8 = gopy version, 9 = backend name, 10 = module file name. +const MakefileTemplateCXX = `# Makefile for python interface for package %[1]s, using %[9]s. +# File is generated by gopy version %[8]s. Do not edit. +# %[2]s + +GOCMD=go +GOBUILD=$(GOCMD) build -mod=mod +GOIMPORTS=goimports +PYTHON=%[4]s +CXX=%[5]s +%[6]s +all: gen build + +gen: + %[3]s + +build: + # $(shell ...) expands to nothing, rather than failing, if $(PYTHON) can't find %[9]s + @test -n "$(CXXLIBFOUND)" || { echo "%[9]s not found for $(PYTHON) (pip install %[9]s)" >&2; exit 1; } + # goimports is needed to ensure that the imports list is valid + $(GOIMPORTS) -w %[1]s.go + # build %[1]s_go.a from %[1]s.go -- the cgo wrappers to go functions -- as a + # static archive: the module below links it in, rather than loading it + $(GOBUILD) -buildmode=c-archive -o %[1]s_go.a %[1]s.go + # writes %[1]s.cpp, the %[9]s module wrapping it + $(PYTHON) build.py + # compile and link %[10]s, the module %[1]s.py imports + $(CXX) %[7]s + +` + +// genMakefileCXX writes the Makefile for the C++ backends. +func (g *pyGen) genMakefileCXX(gencmd string, pycfg PyConfig) { + // The C++ library's own include directories and sources are looked up + // when make runs, not now, so that gopy gen works without it installed. + var libvars string + switch { + case g.isPyBind11(): + libvars = `CXXLIBFLAGS=$(shell $(PYTHON) -m pybind11 --includes) +CXXLIBSRCS= +CXXLIBFOUND=$(CXXLIBFLAGS) +` + case g.isNanobind(): + libvars = `NANOBIND_INC=$(shell $(PYTHON) -c "import nanobind; print(nanobind.include_dir())") +NANOBIND_SRC=$(shell $(PYTHON) -c "import nanobind; print(nanobind.source_dir())") +# robin_map is a dependency nanobind vendors next to its own headers +CXXLIBFLAGS=-I$(NANOBIND_INC) -I$(NANOBIND_INC)/../ext/robin_map/include ` + strings.Join(NanobindCXXFlags, " ") + ` +CXXLIBSRCS=$(NANOBIND_SRC)/nb_combined.cpp +CXXLIBFOUND=$(NANOBIND_INC) +` + } + modlib := ExtModuleName(g.cfg.Name, g.libext, pycfg) + args := CXXArgs(g.cfg.Name, modlib, pycfg, []string{"$(CXXLIBFLAGS)"}, []string{"$(CXXLIBSRCS)"}) + for i, a := range args { + args[i] = makeShellArg(a) + } + // make runs a $(shell ...) or recipe line through sh rather than + // directly whenever it has quotes or other shell syntax in it (as the + // nanobind lookups above do), and sh would strip a Windows path's + // backslashes from $(PYTHON); forward slashes work either way. + vm := filepath.ToSlash(g.cfg.VM) + g.makefile.Printf(MakefileTemplateCXX, g.cfg.Name, g.cfg.Cmd, gencmd, vm, CXX(), libvars, + strings.Join(args, " "), g.cfg.Version, g.cfg.Backend, modlib) +} diff --git a/bind/cxxbuild_test.go b/bind/cxxbuild_test.go new file mode 100644 index 00000000..80965aa9 --- /dev/null +++ b/bind/cxxbuild_test.go @@ -0,0 +1,66 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +var makeShellArgTests = []struct { + arg, want string +}{ + {"", "''"}, + {"-O2", "-O2"}, + {"-I/usr/include/python3.12", "-I/usr/include/python3.12"}, + {"$(CXXLIBFLAGS)", "$(CXXLIBFLAGS)"}, // a make variable, left for make + {"-Wl,-rpath,$ORIGIN", "'-Wl,-rpath,$$ORIGIN'"}, + {"a$b", "'a$$b'"}, + {"-I/opt/My Python/include", "'-I/opt/My Python/include'"}, + {`C:\hostedtoolcache\Python\include`, `'C:\hostedtoolcache\Python\include'`}, + {"it's", `'it'"'"'s'`}, + {`say "hi"`, `'say "hi"'`}, + {"*.cpp", "'*.cpp'"}, + {"a;b", "'a;b'"}, + {"#x", "'#x'"}, +} + +func TestMakeShellArg(t *testing.T) { + for _, tt := range makeShellArgTests { + if got := makeShellArg(tt.arg); got != tt.want { + t.Errorf("makeShellArg(%q) = %q, want %q", tt.arg, got, tt.want) + } + } +} + +// TestMakeShellArgRoundTrip checks makeShellArg's actual contract: written +// into a recipe line, the shell that make runs it with receives arg itself. +func TestMakeShellArgRoundTrip(t *testing.T) { + if _, err := exec.LookPath("make"); err != nil { + t.Skip("make not found") + } + for _, tt := range makeShellArgTests { + if tt.arg == "$(CXXLIBFLAGS)" { + continue // expanded by make, by design + } + dir := t.TempDir() + mk := "all:\n\t@printf '%s' " + makeShellArg(tt.arg) + "\n" + if err := os.WriteFile(filepath.Join(dir, "Makefile"), []byte(mk), 0644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("make", "-s") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("make for %q: %v\n%s", tt.arg, err, out) + continue + } + if string(out) != tt.arg { + t.Errorf("make passed %q to the shell as %q", tt.arg, out) + } + } +} diff --git a/bind/gen.go b/bind/gen.go index faa6e959..98d4fe7c 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -335,6 +335,12 @@ except ImportError: cwd = os.getcwd() currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) os.chdir(currentdir) +# Windows only searches a DLL's own directory for its further dependencies if +# that directory was added explicitly. No backend's module currently depends +# on another DLL next to it (the pybind11 backend's once needed %[1]s_go.pyd, +# before it linked into a single module); harmless, and a no-op elsewhere. +if hasattr(os, 'add_dll_directory'): + os.add_dll_directory(currentdir) # When multiple gopy extensions coexist in one Python process each carries its own # independent Go runtime. Loading each extension without RTLD_GLOBAL below keeps its # Go runtime symbols (including the per-runtime goroutine-pointer TLS slot) local to @@ -431,10 +437,10 @@ def Init(): ` - // 3 = gencmd, 4 = vm, 5 = libext 6 = extraGccArgs, 7 = CFLAGS, 8 = LDLFAGS, - // 9 = windows special declspec hack, 10 = gopy version + // 3 = gencmd, 4 = vm, 5 = libext, 6 = module file name, + // 7 = windows special declspec hack, 8 = gopy version MakefileTemplate = `# Makefile for python interface for package %[1]s. -# File is generated by gopy version %[10]s. Do not edit. +# File is generated by gopy version %[8]s. Do not edit. # %[2]s GOCMD=go @@ -443,11 +449,6 @@ GOIMPORTS=goimports PYTHON=%[4]s LIBEXT=%[5]s -# get the CC and flags used to build python: -GCC = $(shell $(GOCMD) env CC) -CFLAGS = %[7]s -LDFLAGS = %[8]s - all: gen build gen: @@ -459,16 +460,46 @@ build: - rm %[1]s.c # goimports is needed to ensure that the imports list is valid $(GOIMPORTS) -w %[1]s.go - # generate %[1]s_go$(LIBEXT) from %[1]s.go -- the cgo wrappers to go functions + # build %[1]s.go -- the cgo wrappers to go functions -- once, only for the + # %[1]s_go.h header it writes, which %[1]s.c includes $(GOBUILD) -buildmode=c-shared -o %[1]s_go$(LIBEXT) %[1]s.go - # use pybindgen to build the %[1]s.c file which are the CPython wrappers to cgo wrappers.. - # note: pip install pybindgen to get pybindgen if this fails + - rm %[1]s_go$(LIBEXT) + # write %[1]s.c, the CPython wrappers to the cgo wrappers + $(PYTHON) build.py%[7]s + # build the module %[1]s.py imports: cgo compiles %[1]s.c into the same + # shared library as the go code, so it depends on nothing but python + $(GOBUILD) -buildmode=c-shared -o %[6]s . + +` + + // 1 = package name, 2 = gopy command, 3 = gencmd, 4 = vm, 5 = libext, + // 10 = gopy version (6-9 are unused): cffi needs no %[1]s.c, and none of + // the CPython flags that building one would. + MakefileTemplateCFFI = `# Makefile for python interface for package %[1]s, using cffi. +# File is generated by gopy version %[10]s. Do not edit. +# %[2]s + +GOCMD=go +GOBUILD=$(GOCMD) build -mod=mod +GOIMPORTS=goimports +PYTHON=%[4]s +LIBEXT=%[5]s + +all: gen build + +gen: + %[3]s + +build: + # goimports is needed to ensure that the imports list is valid + $(GOIMPORTS) -w %[1]s.go + # generate %[1]s_go$(LIBEXT) from %[1]s.go -- the cgo wrappers to go functions. + # unlike the default (pybindgen) backend, this is the only library gopy + # builds: cffi loads it directly, with no Python.h/libpython involved. + $(GOBUILD) -buildmode=c-shared -o %[1]s_go$(LIBEXT) %[1]s.go + # writes _%[1]s.py, the cffi module %[1]s.py imports $(PYTHON) build.py - # build the _%[1]s$(LIBEXT) library that contains the cgo and CPython wrappers - # generated %[1]s.py python wrapper imports this c-code package - %[9]s - $(GCC) %[1]s.c %[6]s %[1]s_go$(LIBEXT) -o _%[1]s$(LIBEXT) $(CFLAGS) $(LDFLAGS) -fPIC --shared -w - + ` // exe version of template: 3 = gencmd, 4 = vm, 5 = libext, 8 = gopy version @@ -541,15 +572,14 @@ var ClearGoTLS = false // and wrapper .py file(s) that are loaded as the interface to the package with shadow // python-side classes // mode = gen, build, pkg, exe -func GenPyBind(mode BuildMode, libext, extragccargs string, lang int, dynamicLink bool, cfg *BindCfg) error { +func GenPyBind(mode BuildMode, libext string, lang int, dynamicLink bool, cfg *BindCfg) error { gen := &pyGen{ - mode: mode, - pypkgname: cfg.Name, - cfg: cfg, - libext: libext, - extraGccArgs: extragccargs, - lang: lang, - dynamicLink: dynamicLink, + mode: mode, + pypkgname: cfg.Name, + cfg: cfg, + libext: libext, + lang: lang, + dynamicLink: dynamicLink, } gen.genPackageMap() thePyGen = gen @@ -572,13 +602,13 @@ type pyGen struct { err ErrorList pkgmap map[string]struct{} // map of package paths - mode BuildMode // mode: gen, build, pkg, exe - pypkgname string - cfg *BindCfg - libext string - extraGccArgs string - lang int // c-python api version (2,3) - dynamicLink bool + mode BuildMode // mode: gen, build, pkg, exe + pypkgname string + cfg *BindCfg + libext string + lang int // c-python api version (2,3) + dynamicLink bool + cbs []*cffiCallback // cffi: the callback types that have a C trampoline, see cffi_callback.go } func (g *pyGen) gen() error { @@ -635,8 +665,15 @@ func (g *pyGen) genPrintOut(outfn string, pr *printer) { } func (g *pyGen) genOut() { - g.pybuild.Printf("\nmod.generate(open('%v.c', 'w'))\n\n", g.cfg.Name) + switch { + case g.noAPIShim(): + g.pybuild.Printf("\nmod.generate()\n\n") + default: + g.pybuild.Printf("\nmod.generate(open('%v.c', 'w'))\n\n", g.cfg.Name) + } g.gofile.Printf("\n\n") + g.spliceCFFITrampolines() + g.splicePyBind11Trampolines() g.genPrintOut(g.cfg.Name+".go", g.gofile) g.genPrintOut("build.py", g.pybuild) if !NoMake { @@ -691,6 +728,19 @@ func (g *pyGen) genGoPreamble() { pkgimport += fmt.Sprintf("\n\t%q", pp) } } + if g.noAPIShim() { + trampolines := "" + switch { + case g.isCFFI(): + trampolines = cffiTrampolinesKey + case g.isCXXShim(): + trampolines = pybind11TrampolinesKey + } + g.gofile.Printf(goPreambleCFFI, g.cfg.Name, g.cfg.Cmd, "", GoHandle, CGoHandle, + pkgimport, g.cfg.Main, trampolines, "", g.cfg.Version) + g.gofile.Printf("\n// --- generated code for package: %[1]s below: ---\n\n", g.cfg.Name) + return + } libcfg := func() string { pycfg, err := GetPythonConfig(g.cfg.VM) if err != nil { @@ -727,7 +777,30 @@ func (g *pyGen) genGoPreamble() { } func (g *pyGen) genPyBuildPreamble() { - g.pybuild.Printf(PyBuildPreamble, g.cfg.Name, g.cfg.Cmd, g.cfg.Version) + switch { + case g.isCFFI(): + g.pybuild.Printf("%s", g.buildPreamble(cffiBuildPy, "@LIBEXT@", g.libext)) + case g.isPyBind11(): + g.pybuild.Printf("%s", g.buildPreamble(pybind11BuildPy, "@ARG_HELPERS@", cxxArgHelpers)) + case g.isNanobind(): + g.pybuild.Printf("%s", g.buildPreamble(nanobindBuildPy, "@ARG_HELPERS@", cxxArgHelpers)) + case g.isCAPI(): + g.pybuild.Printf("%s", g.buildPreamble(capiBuildPy)) + default: + g.pybuild.Printf(PyBuildPreamble, g.cfg.Name, g.cfg.Cmd, g.cfg.Version) + } +} + +// buildPreamble returns src, the start of build.py for a backend whose +// recorder is an embedded .py file, with its @NAME@, @CMD@ and @VERSION@ +// filled in, along with any of its own placeholders that extra gives as +// placeholder, value pairs. +func (g *pyGen) buildPreamble(src string, extra ...string) string { + return strings.NewReplacer(append([]string{ + "@NAME@", g.cfg.Name, + "@CMD@", g.cfg.Cmd, + "@VERSION@", g.cfg.Version, + }, extra...)...).Replace(src) } func (g *pyGen) genPyWrapPreamble() { @@ -818,13 +891,19 @@ func (g *pyGen) genMakefile() { if g.mode == ModeExe { g.makefile.Printf(MakefileExeTemplate, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, pycfg.CFlags, pycfg.LdFlags, g.cfg.Version) + } else if g.isCXXShim() { + g.genMakefileCXX(gencmd, pycfg) + } else if g.isCFFI() { + g.makefile.Printf(MakefileTemplateCFFI, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, "", "", "", "", g.cfg.Version) } else { winhack := "" if WindowsOS { - winhack = fmt.Sprintf(`# windows-only sed hack here to fix pybindgen declaration of PyInit - sed -i "s/ PyInit_/ __declspec(dllexport) PyInit_/g" %s.c`, g.cfg.Name) + winhack = fmt.Sprintf(` + # windows-only sed hack here to fix pybindgen declaration of PyInit + sed -i "s/ PyInit_/ __declspec(dllexport) PyInit_/g" %s.c`, g.cfg.Name) } - g.makefile.Printf(MakefileTemplate, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, g.extraGccArgs, pycfg.CFlags, pycfg.LdFlags, winhack, g.cfg.Version) + g.makefile.Printf(MakefileTemplate, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, + ExtModuleName(g.cfg.Name, g.libext, pycfg), winhack, g.cfg.Version) } } diff --git a/bind/gen_func.go b/bind/gen_func.go index b2643343..33d14082 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -67,6 +67,31 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { return false } + // None of the no-API backends (cffi, pybind11, nanobind) can cross a raw + // PyObject* (complex64/128, and a callback argument of a type + // cffiCallback doesn't handle): + // skip these functions rather than emit a signature referencing the + // CPython C API, which would fail to even compile under their preamble. + if g.noAPIShim() { + for _, arg := range args { + sarg := current.symtype(arg.GoType()) + switch { + case sarg == nil: + case sarg.isSignature(): + if g.cffiCallback(sarg) == nil { + return false + } + case sarg.cpyname == "PyObject*": + return false + } + } + for _, ret := range res { + if sret := current.symtype(ret.GoType()); sret != nil && sret.cpyname == "PyObject*" { + return false + } + } + } + var ( goArgs []string pyArgs []string @@ -88,10 +113,17 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { } anm := pySafeArg(arg.Name(), i) - if ifchandle && arg.sym.goname == "interface{}" { + switch { + case g.isCFFI() && sarg.isSignature(): + goArgs = append(goArgs, fmt.Sprintf("%s unsafe.Pointer", anm)) + pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) + case g.isCXXShim() && sarg.isSignature(): + goArgs = append(goArgs, fmt.Sprintf("%s CGoHandle", anm)) + pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) + case ifchandle && arg.sym.goname == "interface{}": goArgs = append(goArgs, fmt.Sprintf("%s %s", anm, CGoHandle)) pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", PyHandle, anm)) - } else { + default: goArgs = append(goArgs, fmt.Sprintf("%s %s", anm, sarg.cgoname)) if sarg.cpyname == "PyObject*" { pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s', transfer_ownership=False)", sarg.cpyname, anm)) @@ -196,11 +228,77 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { } func (g *pyGen) genFunc(o *Func) { + if g.noAPIShim() && g.genFuncComplexCFFI(o) { + return + } if g.genFuncSig(nil, o) { g.genFuncBody(nil, o) } } +// genFuncComplexCFFI generates a plain (non-method) function whose every +// argument and its one return value are complex64/128, which the normal path +// (genFuncSig/genFuncBody) can't do under cffi, where a complex value crosses +// as two floats (see isComplexShim in cffi.go). It returns false, writing +// nothing, if the signature doesn't fit that narrow shape (methods, a mix of +// complex and other argument types, or an error return): genFuncSig's +// PyObject* check then skips the function instead of emitting code that +// fails to compile. +func (g *pyGen) genFuncComplexCFFI(fsym *Func) bool { + sig := fsym.sig + if sig == nil || fsym.isVariadic || fsym.err { + return false + } + args := sig.Params() + res := sig.Results() + if len(res) != 1 || !isComplexSym(current.symtype(res[0].GoType())) { + return false + } + for _, arg := range args { + if !isComplexSym(current.symtype(arg.GoType())) { + return false + } + } + + gname := fsym.GoName() + if g.cfg.RenameCase { + gname = toSnakeCase(gname) + } + gname, gdoc, err := extractPythonName(gname, fsym.Doc()) + if err != nil { + return false + } + + ret := current.symtype(res[0].GoType()) + var goArgs, pyArgs, callArgs, wpArgs []string + for i, arg := range args { + anm := pySafeArg(arg.Name(), i) + sarg := current.symtype(arg.GoType()) + goArgs = append(goArgs, g.cgoParam(anm, sarg)) + pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cpyName(sarg), anm)) + callArgs = append(callArgs, g.cgoToGo(sarg, anm)) + wpArgs = append(wpArgs, anm) + } + + g.gofile.Printf("\n//export %s\n", fsym.ID()) + g.gofile.Printf("func %s(%s) %s {\n", fsym.ID(), strings.Join(goArgs, ", "), g.cgoResult(ret)) + g.gofile.Indent() + g.gofile.Printf("return %s\n", g.goToCgo(ret, fmt.Sprintf("%s(%s)", fsym.GoFmt(), strings.Join(callArgs, ", ")))) + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.pybuild.Printf("mod.add_function('%s', retval('%s'), [%s])\n", fsym.ID(), g.cpyName(ret), strings.Join(pyArgs, ", ")) + + g.pywrap.Printf("def %s(%s):\n", gname, strings.Join(wpArgs, ", ")) + g.pywrap.Indent() + g.pywrap.Printf(`"""%s"""`, gdoc) + g.pywrap.Printf("\n") + g.pywrap.Printf("return _%s.%s(%s)\n", g.cfg.Name, fsym.ID(), strings.Join(wpArgs, ", ")) + g.pywrap.Outdent() + + return true +} + func (g *pyGen) genMethod(s *symbol, o *Func) { if g.genFuncSig(s, o) { g.genFuncBody(s, o) @@ -255,15 +353,26 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { g.gofile.Indent() if fsym.hasfun { for i, arg := range args { - if arg.sym.isSignature() { + switch { + case arg.sym.isSignature() && g.isCFFI(): + g.gofile.Printf("%s", cffiCallbackPrologue(pySafeArg(arg.Name(), i))) + case arg.sym.isSignature() && g.isCXXShim(): + // no Go-side setup: the C++ registry itself refuses a call + // once the wrapping python call unregisters it (see + // pybind11_callback.go) + case arg.sym.isSignature(): g.gofile.Printf("_fun_arg := %s\n", pySafeArg(arg.Name(), i)) } } } - g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n") - if !rvIsErr && nres != 2 { - g.gofile.Printf("defer C.PyEval_RestoreThread(_saved_thread)\n") + // cffi, pybind11 and nanobind each release the GIL themselves around + // every call + if !g.noAPIShim() { + g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n") + if !rvIsErr && nres != 2 { + g.gofile.Printf("defer C.PyEval_RestoreThread(_saved_thread)\n") + } } if isMethod { @@ -302,6 +411,10 @@ if __err != nil { switch { case ifchandle && arg.sym.goname == "interface{}": na = fmt.Sprintf(`gopyh.VarFromHandle((gopyh.CGoHandle)(%s), "interface{}")`, anm) + case arg.sym.isSignature() && g.isCFFI(): + na = g.cffiCallbackLit(g.cffiCallback(arg.sym), anm) + case arg.sym.isSignature() && g.isCXXShim(): + na = g.pybind11CallbackLit(g.cffiCallback(arg.sym), anm) case arg.sym.isSignature(): na = fmt.Sprintf("%s", arg.sym.py2go) case arg.sym.py2go != "": @@ -424,12 +537,18 @@ if __err != nil { if rvIsErr || nres == 2 { g.gofile.Printf("\n") - g.gofile.Printf("C.PyEval_RestoreThread(_saved_thread)\n") + if !g.noAPIShim() { + g.gofile.Printf("C.PyEval_RestoreThread(_saved_thread)\n") + } g.gofile.Printf("if __err != nil {\n") g.gofile.Indent() g.gofile.Printf("estr := C.CString(__err.Error())\n") - g.gofile.Printf("C.PyErr_SetString(C.PyExc_RuntimeError, estr)\n") + if g.noAPIShim() { + g.gofile.Printf("%s", g.goSetError("RuntimeError", "__err.Error()")) + } else { + g.gofile.Printf("C.PyErr_SetString(C.PyExc_RuntimeError, estr)\n") + } if rvIsErr { g.gofile.Printf("return estr\n") // NOTE: leaked string } else { diff --git a/bind/gen_map.go b/bind/gen_map.go index 06d3302d..5da36a71 100644 --- a/bind/gen_map.go +++ b/bind/gen_map.go @@ -299,7 +299,11 @@ otherwise parameter is a python list that we copy from } g.gofile.Printf("if !ok {\n") g.gofile.Indent() - g.gofile.Printf("C.PyErr_SetString(C.PyExc_KeyError, C.CString(\"key not in map\"))\n") + if g.noAPIShim() { + g.gofile.Printf("%s", g.goSetError("KeyError", `"key not in map"`)) + } else { + g.gofile.Printf("C.PyErr_SetString(C.PyExc_KeyError, C.CString(\"key not in map\"))\n") + } g.gofile.Outdent() g.gofile.Printf("}\n") if esym.go2py != "" { diff --git a/bind/gen_slice.go b/bind/gen_slice.go index 1c16180c..ee1c9127 100644 --- a/bind/gen_slice.go +++ b/bind/gen_slice.go @@ -70,6 +70,14 @@ func (g *pyGen) genSliceInit(slc *symbol, extTypes, pyWrapOnly bool, slob *Slice esym = current.symtype(typ.Elem()) } + // element access (elem/set/append) below would reference *C.PyObject, + // which cffi's preamble doesn't declare (see genFuncSig for the same + // restriction on plain function args/returns); skip the whole wrapper + // rather than emit code that fails to compile. + if g.noAPIShim() && esym != nil && esym.cpyname == "PyObject*" && !g.isComplexShim(esym) { + return + } + gocl := "go." if g.pkg == goPackage { gocl = "" @@ -321,21 +329,15 @@ otherwise parameter is a python list that we copy from g.pybuild.Printf("mod.add_function('%s_len', retval('int'), [param('%s', 'handle')])\n", slNm, PyHandle) g.gofile.Printf("//export %s_elem\n", slNm) - g.gofile.Printf("func %s_elem(handle CGoHandle, _idx int) %s {\n", slNm, esym.cgoname) + g.gofile.Printf("func %s_elem(handle CGoHandle, _idx int) %s {\n", slNm, g.cgoResult(esym)) g.gofile.Indent() g.gofile.Printf("s := deptrFromHandle_%s(handle)\n", slNm) - if esym.go2py != "" { - // If the go2py starts with handleFromPtr_, use reference &, otherwise just return the value - val_str := "" - if strings.HasPrefix(esym.go2py, "handleFromPtr_") { - val_str = "&(s[_idx])" - } else { - val_str = "s[_idx]" - } - g.gofile.Printf("return %s(%s)%s\n", esym.go2py, val_str, esym.go2pyParenEx) - } else { - g.gofile.Printf("return s[_idx]\n") + // If the go2py starts with handleFromPtr_, use reference &, otherwise just return the value + val_str := "s[_idx]" + if strings.HasPrefix(esym.go2py, "handleFromPtr_") { + val_str = "&(s[_idx])" } + g.gofile.Printf("return %s\n", g.goToCgo(esym, val_str)) g.gofile.Outdent() g.gofile.Printf("}\n\n") @@ -348,7 +350,7 @@ otherwise parameter is a python list that we copy from if esym.cpyname == "char*" { g.pybuild.Printf("add_checked_string_function(mod, '%s_elem', retval('%s'), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, PyHandle) } else { - g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'%s), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, caller_owns_ret, PyHandle) + g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'%s), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, g.cpyName(esym), caller_owns_ret, PyHandle) } if slc.isSlice() { @@ -365,64 +367,111 @@ otherwise parameter is a python list that we copy from } g.gofile.Printf("//export %s_set\n", slNm) - g.gofile.Printf("func %s_set(handle CGoHandle, _idx int, _vl %s) {\n", slNm, esym.cgoname) + g.gofile.Printf("func %s_set(handle CGoHandle, _idx int, %s) {\n", slNm, g.cgoParam("_vl", esym)) g.gofile.Indent() g.gofile.Printf("s := deptrFromHandle_%s(handle)\n", slNm) - if esym.py2go != "" { - g.gofile.Printf("s[_idx] = %s(_vl)%s\n", esym.py2go, esym.py2goParenEx) - } else { - g.gofile.Printf("s[_idx] = _vl\n") - } + g.gofile.Printf("s[_idx] = %s\n", g.cgoToGo(esym, "_vl")) g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_set', None, [param('%s', 'handle'), param('int', 'idx'), param('%v', 'value'%s)])\n", slNm, PyHandle, esym.cpyname, transfer_ownership) + g.pybuild.Printf("mod.add_function('%s_set', None, [param('%s', 'handle'), param('int', 'idx'), param('%v', 'value'%s)])\n", slNm, PyHandle, g.cpyName(esym), transfer_ownership) if slc.isSlice() { g.gofile.Printf("//export %s_append\n", slNm) - g.gofile.Printf("func %s_append(handle CGoHandle, _vl %s) {\n", slNm, esym.cgoname) + g.gofile.Printf("func %s_append(handle CGoHandle, %s) {\n", slNm, g.cgoParam("_vl", esym)) g.gofile.Indent() g.gofile.Printf("s := ptrFromHandle_%s(handle)\n", slNm) - if esym.py2go != "" { - g.gofile.Printf("*s = append(*s, %s(_vl)%s)\n", esym.py2go, esym.py2goParenEx) - } else { - g.gofile.Printf("*s = append(*s, _vl)\n") - } + g.gofile.Printf("*s = append(*s, %s)\n", g.cgoToGo(esym, "_vl")) g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_append', None, [param('%s', 'handle'), param('%s', 'value'%s)])\n", slNm, PyHandle, esym.cpyname, transfer_ownership) + g.pybuild.Printf("mod.add_function('%s_append', None, [param('%s', 'handle'), param('%s', 'value'%s)])\n", slNm, PyHandle, g.cpyName(esym), transfer_ownership) } if slNm == "Slice_byte" { - g.gofile.Printf("//export Slice_byte_from_bytes\n") - g.gofile.Printf("func Slice_byte_from_bytes(o *C.PyObject) CGoHandle {\n") - g.gofile.Indent() - g.gofile.Printf("size := C.PyBytes_Size(o)\n") - g.gofile.Printf("ptr := unsafe.Pointer(C.PyBytes_AsString(o))\n") - g.gofile.Printf("data := make([]byte, size)\n") - g.gofile.Printf("tmp := unsafe.Slice((*byte)(ptr), size)\n") - g.gofile.Printf("copy(data, tmp)\n") - g.gofile.Printf("return handleFromPtr_Slice_byte(&data)\n") - g.gofile.Outdent() - g.gofile.Printf("}\n\n") - - g.gofile.Printf("//export Slice_byte_to_bytes\n") - g.gofile.Printf("func Slice_byte_to_bytes(handle CGoHandle) *C.PyObject {\n") - g.gofile.Indent() - g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") - g.gofile.Printf("ptr := unsafe.Pointer(&s[0])\n") - g.gofile.Printf("size := len(s)\n") - if WindowsOS { - g.gofile.Printf("return C.PyBytes_FromStringAndSize((*C.char)(ptr), C.longlong(size))\n") + if g.noAPIShim() { + // PyBytes_* is off-limits for cffi (no CPython headers), so these + // exchange a raw pointer+length instead of a PyObject*; the cffi + // build script (cffi_build.py) recognizes them by name and writes + // the bytes<->buffer conversion into the generated python module. + g.gofile.Printf("//export Slice_byte_from_bytes\n") + g.gofile.Printf("func Slice_byte_from_bytes(ptr unsafe.Pointer, size C.longlong) CGoHandle {\n") + g.gofile.Indent() + g.gofile.Printf("data := make([]byte, size)\n") + g.gofile.Printf("if size > 0 {\n") + g.gofile.Indent() + g.gofile.Printf("tmp := unsafe.Slice((*byte)(ptr), size)\n") + g.gofile.Printf("copy(data, tmp)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n") + g.gofile.Printf("return handleFromPtr_Slice_byte(&data)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.gofile.Printf("//export Slice_byte_to_bytes_len\n") + g.gofile.Printf("func Slice_byte_to_bytes_len(handle CGoHandle) C.longlong {\n") + g.gofile.Indent() + g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") + g.gofile.Printf("return C.longlong(len(s))\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + // Returning &s[0] directly would hand cgo a pointer into the Go + // heap, which cgo's pointer checks reject once it crosses back + // to the caller; copy into a C-owned buffer instead, freed by + // the python side (Slice_byte_free_ptr) once it has read it. + g.gofile.Printf("//export Slice_byte_to_bytes_ptr\n") + g.gofile.Printf("func Slice_byte_to_bytes_ptr(handle CGoHandle) unsafe.Pointer {\n") + g.gofile.Indent() + g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") + g.gofile.Printf("n := len(s)\n") + g.gofile.Printf("if n == 0 {\n") + g.gofile.Indent() + g.gofile.Printf("return nil\n") + g.gofile.Outdent() + g.gofile.Printf("}\n") + g.gofile.Printf("buf := C.malloc(C.size_t(n))\n") + g.gofile.Printf("copy(unsafe.Slice((*byte)(buf), n), s)\n") + g.gofile.Printf("return buf\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.gofile.Printf("//export Slice_byte_free_ptr\n") + g.gofile.Printf("func Slice_byte_free_ptr(ptr unsafe.Pointer) {\n") + g.gofile.Indent() + g.gofile.Printf("C.free(ptr)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") } else { - g.gofile.Printf("return C.PyBytes_FromStringAndSize((*C.char)(ptr), C.long(size))\n") - } - g.gofile.Outdent() - g.gofile.Printf("}\n\n") + g.gofile.Printf("//export Slice_byte_from_bytes\n") + g.gofile.Printf("func Slice_byte_from_bytes(o *C.PyObject) CGoHandle {\n") + g.gofile.Indent() + g.gofile.Printf("size := C.PyBytes_Size(o)\n") + g.gofile.Printf("ptr := unsafe.Pointer(C.PyBytes_AsString(o))\n") + g.gofile.Printf("data := make([]byte, size)\n") + g.gofile.Printf("tmp := unsafe.Slice((*byte)(ptr), size)\n") + g.gofile.Printf("copy(data, tmp)\n") + g.gofile.Printf("return handleFromPtr_Slice_byte(&data)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.gofile.Printf("//export Slice_byte_to_bytes\n") + g.gofile.Printf("func Slice_byte_to_bytes(handle CGoHandle) *C.PyObject {\n") + g.gofile.Indent() + g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") + g.gofile.Printf("ptr := unsafe.Pointer(&s[0])\n") + g.gofile.Printf("size := len(s)\n") + if WindowsOS { + g.gofile.Printf("return C.PyBytes_FromStringAndSize((*C.char)(ptr), C.longlong(size))\n") + } else { + g.gofile.Printf("return C.PyBytes_FromStringAndSize((*C.char)(ptr), C.long(size))\n") + } + g.gofile.Outdent() + g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('Slice_byte_from_bytes', retval('%s'%s), [param('PyObject*', 'o', transfer_ownership=False)])\n", PyHandle, caller_owns_ret) - g.pybuild.Printf("mod.add_function('Slice_byte_to_bytes', retval('PyObject*', caller_owns_return=True), [param('%s', 'handle')])\n", PyHandle) + g.pybuild.Printf("mod.add_function('Slice_byte_from_bytes', retval('%s'%s), [param('PyObject*', 'o', transfer_ownership=False)])\n", PyHandle, caller_owns_ret) + g.pybuild.Printf("mod.add_function('Slice_byte_to_bytes', retval('PyObject*', caller_owns_return=True), [param('%s', 'handle')])\n", PyHandle) + } } } } diff --git a/bind/nanobind.go b/bind/nanobind.go new file mode 100644 index 00000000..6762254f --- /dev/null +++ b/bind/nanobind.go @@ -0,0 +1,19 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" +) + +// The nanobind backend (GOPY_BACKEND=nanobind) is the pybind11 backend with +// a different C++ library on the consumer side: the same cgo shim (see +// noAPIShim), the same handle-registry callbacks (see isCXXShim and +// pybind11_callback.go), the same single final link against a c-archive (see +// buildCXXModule in cmd_build.go). Only nanobind_build.py, a fork of +// pybind11_build.py written against nanobind's API, is its own. + +//go:embed nanobind_build.py +var nanobindBuildPy string diff --git a/bind/nanobind_build.py b/bind/nanobind_build.py new file mode 100644 index 00000000..56aad685 --- /dev/null +++ b/bind/nanobind_build.py @@ -0,0 +1,365 @@ +# python build stub for package @NAME@ (nanobind backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes +# @NAME@.cpp: a nanobind module that #includes @NAME@_go.h (cgo's own header) +# and calls the shim functions directly. This is a fork of pybind11_build.py +# (nanobind's API mirrors pybind11's): keep the two in step, differing only +# where nanobind itself does. + + +def retval(ctype, *a, **kw): + return ctype + + +def param(ctype, name, *a, **kw): + return (ctype, name) + + +class Module(object): + def __init__(self, name, cpp_name): + self.name = name # the compiled extension's import name, e.g. "_hi" + self.cpp_name = cpp_name # source file to write, e.g. "hi.cpp" + self.header = None + self.funcs = [] + + def add_include(self, inc): + self.header = inc.strip('"') + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append((name, ret, params)) + + def generate(self): + import os + + here = os.path.dirname(os.path.abspath(__file__)) + # a callback's C++ trampoline is shared by every callable of the same + # shape (see pybind11_callback.go, which nanobind shares), keyed and + # numbered here in the same first-seen order Go numbered them in, so + # "gopy_cb_" means the same thing on both sides without the two + # ever exchanging it. + callback_kinds = {} + defs = [d for d in (wrapper(name, ret, params, callback_kinds) for name, ret, params in self.funcs) if d] + trampolines = "\n".join(callback_trampoline(ctype, i) for ctype, i in callback_kinds.items()) + cpp = ( + MODULE_TEMPLATE.replace("@HEADER@", self.header) + .replace("@CALLBACK_TRAMPOLINES@", trampolines) + .replace("@DEFS@", "\n".join(defs)) + ) + with open(os.path.join(here, self.cpp_name), "w") as f: + f.write(cpp) + + +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.add_function(name, retval, params) + + +add_checked_string_function = add_checked_function + + +# How wrapper converts an argument of each C type: as pybindgen's generated +# code would, for the same exception on a wrongly-typed argument under every +# backend (see the gopy_arg_* helpers in cxx_args.inc). %(o)s is the +# argument's PyObject*, %(n)d its 1-based position. Identical in +# pybind11_build.py and nanobind_build.py. +ARG_CONV = { + "int64_t": "gopy_arg_L(%(o)s)", + "long long": "gopy_arg_L(%(o)s)", + "uint64_t": "gopy_arg_K(%(o)s, %(n)d)", + "int": "gopy_arg_i(%(o)s)", + "int32_t": "gopy_arg_i(%(o)s)", + "uint32_t": "gopy_arg_I(%(o)s)", + "unsigned int": "gopy_arg_I(%(o)s)", + "int16_t": "(int16_t)gopy_arg_i_max(%(o)s, 0x7fff)", + "uint16_t": "(uint16_t)gopy_arg_i_max(%(o)s, 0xffff)", + "int8_t": "(int8_t)gopy_arg_i_max(%(o)s, 0x7f)", + "uint8_t": "(uint8_t)gopy_arg_i_max(%(o)s, 0xff)", + "double": "gopy_arg_d(%(o)s)", + "float": "(float)gopy_arg_d(%(o)s)", + "bool": "gopy_arg_bool(%(o)s)", + "char*": "const_cast(gopy_arg_str(%(o)s, %(n)d))", +} + + +def wrapper(name, ret, params, callback_kinds): + """Returns the m.def(...) call binding name, or "" if its signature + isn't supported yet (a raw PyObject*): the .cpp simply never binds it, so + calling it from python raises AttributeError instead of + NotImplementedError -- close enough for a function nothing in gopy's own + generated wrapper calls unconditionally. callback_kinds is shared across + every call from Module.generate, one entry per distinct callback shape + seen so far (see there and callback_trampoline). + """ + if ret == "PyObject*" or any(p[0] == "PyObject*" for p in params): + return "" + args = [] + setup = [] + call_args = [] + for i, (ctype, pname) in enumerate(params): + if ctype in ARG_CONV: + # converted up front, in order, while the GIL is still held + args.append("nb::handle " + pname) + setup.append( + "auto _a_%s = %s;" % (pname, ARG_CONV[ctype] % {"o": pname + ".ptr()", "n": i + 1}) + ) + call_args.append("_a_" + pname) + elif ctype in ("complex64", "complex128"): + cxxfloat = "float" if ctype == "complex64" else "double" + args.append("std::complex<%s> %s" % (cxxfloat, pname)) + call_args.append("%s.real(), %s.imag()" % (pname, pname)) + elif ctype.startswith("callback:"): + i = callback_kinds.setdefault(ctype, len(callback_kinds)) + args.append("nb::handle " + pname) + setup.append( + "gopy_arg_callable(%s.ptr(), %d);\n" + " int64_t _h_%s = gopy_cb_register(nb::borrow(%s));\n" + " GopyCBGuard _g_%s{_h_%s};" % (pname, i + 1, pname, pname, pname, pname) + ) + call_args.append("_h_%s" % pname) + else: + args.append(ctype + " " + pname) + call_args.append(pname) + call = "%s(%s)" % (name, ", ".join(call_args)) + # Releasing the GIL only around the call itself (not the setup/result + # handling around it, which need it) matches what cffi gets for free + # from ctypes/cffi's own default behavior, and is what makes a callback + # arrive correctly rather than deadlock: Go may run it from a goroutine + # (see InGoroutine in _examples/callbacks) while this call's own thread + # blocks waiting for that goroutine, so it must not be left holding the + # only GIL there is. + if ret is None: + call = "[&]{ nb::gil_scoped_release _rel; %s; }()" % call + else: + call = "[&]{ nb::gil_scoped_release _rel; return %s; }()" % call + if ret is None: + body, cpptype = "%s;\n _check();" % call, "void" + elif ret == "char*": + body = ( + "char* _r = %s;\n" + " std::string _s(_r ? _r : \"\");\n" + " free(_r);\n" + " _check();\n" + " return _s;" % call + ) + cpptype = "std::string" + elif ret == "bool": + body = "auto _r = %s;\n _check();\n return _r != 0;" % call + cpptype = "bool" + elif ret in ("complex64", "complex128"): + cxxfloat = "float" if ret == "complex64" else "double" + cpptype = "std::complex<%s>" % cxxfloat + body = ( + "auto _r = %s;\n" + " _check();\n" + " return %s(_r.r0, _r.r1);" % (call, cpptype) + ) + else: + body = "auto _r = %s;\n _check();\n return _r;" % call + cpptype = ret + if setup: + body = "\n ".join(setup) + "\n " + body + # Unlike pybind11, nanobind refuses None for an nb::handle parameter + # unless told otherwise, before the gopy_arg_* helper that should report + # it (as pybindgen would) ever runs; telling it so takes an nb::arg for + # every parameter, not just those. + annotations = "".join( + ', nb::arg("%s")%s' % (a.rsplit(" ", 1)[1], ".none()" if a.startswith("nb::handle ") else "") + for a in args + ) + return ' m.def("%s", [](%s) -> %s {\n %s\n }%s);' % ( + name, + ", ".join(args), + cpptype, + body, + annotations, + ) + + +def callback_trampoline(ctype, idx): + """Returns the static gopy_cb_ trampoline for the callback shape in + ctype ("callback:()", see cffiCallback + in cffi_callback.go): the Go closure for every callable of this shape + calls gopy_cb_, passing its own registry handle as the first + argument (see pybind11CallbackLit in pybind11_callback.go). + """ + ret, _, rest = ctype[len("callback:") :].partition("(") + ctypes_ = [t for t in rest[:-1].split(",") if t] + names = ["a%d" % i for i in range(len(ctypes_))] + + def cxxparam(t): + return "unsigned char" if t == "bool" else t + + params = "".join(", %s %s" % (cxxparam(t), n) for t, n in zip(ctypes_, names)) + call_args = [] + for t, n in zip(ctypes_, names): + if t == "char*": + call_args.append("nb::str(%s ? %s : \"\")" % (n, n)) + elif t == "bool": + call_args.append("nb::bool_(%s != 0)" % n) + else: + call_args.append(n) + call = "fn(%s)" % ", ".join(call_args) + cxxret = "void" if ret == "void" else cxxparam(ret) + zero = "" if ret == "void" else " 0" + if ret == "void": + body = "%s;" % call + elif ret == "bool": + body = "return nb::cast(%s) ? 1 : 0;" % call + else: + body = "return nb::cast<%s>(%s);" % (ret, call) + # A raised exception must not reach the extern "C" boundary as a C++ + # exception: unwinding through Go's compiled call frames is undefined + # behavior (a hard crash in practice). Printing it and returning the + # zero value instead matches what cffi's ffi.callback does by default. + body = ( + "try {\n" + " %s\n" + " } catch (nb::python_error& e) {\n" + " e.restore();\n" + " PyErr_Print();\n" + " return%s;\n" + " }" % (body, zero) + ) + return ( + # gil must be declared (and so acquired) before fn: C++ destroys + # locals in reverse declaration order, and fn (an nb::callable) needs + # the GIL held for its own destructor -- declared the other way + # around, gil would release it first, and fn would decref without it. + 'extern "C" %s gopy_cb_%d(int64_t h%s) {\n' + " nb::gil_scoped_acquire gil;\n" + " nb::callable fn;\n" + " if (!gopy_cb_lookup(h, fn)) {\n" + " return%s;\n" + " }\n" + " %s\n" + "}" % (cxxret, idx, params, zero, body) + ) + + +MODULE_TEMPLATE = '''// python bindings for package @NAME@ using nanobind. +// File is generated by gopy version @VERSION@. Do not edit. +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace nb = nanobind; + +extern "C" { +#include "@HEADER@" +} + +// Hands the python error already set back to python, from C++. +[[noreturn]] static void gopy_raise() { + throw nb::python_error(); +} + +@ARG_HELPERS@ + +// Raises the exception, if any, that the last Go call on this thread +// recorded (see gopySetError/GopyTakeError, shared with the cffi backend). +static inline void _check() { + char* e = GopyTakeError(); + if (!e) { + return; + } + std::string s(e); + free(e); + auto sep = s.find(':'); + std::string kind = sep == std::string::npos ? s : s.substr(0, sep); + std::string msg = sep == std::string::npos ? "" : s.substr(sep + 1); + PyObject* exc = PyExc_RuntimeError; + if (kind == "ValueError") exc = PyExc_ValueError; + else if (kind == "TypeError") exc = PyExc_TypeError; + else if (kind == "KeyError") exc = PyExc_KeyError; + else if (kind == "IndexError") exc = PyExc_IndexError; + else if (kind == "AttributeError") exc = PyExc_AttributeError; + PyErr_SetString(exc, msg.c_str()); + throw nb::python_error(); +} + +// A python callable passed as a func-typed argument is registered here for +// the duration of the call it was passed to (see pybind11_callback.go for +// why a registry rather than one C function pointer per callable), and the +// trampolines below (one per callback shape, see callback_trampoline in +// nanobind_build.py) look it up by handle each time Go calls back in. +static std::mutex gopy_cb_mutex; +static std::unordered_map gopy_cb_registry; +static int64_t gopy_cb_next = 1; + +static int64_t gopy_cb_register(nb::callable fn) { + std::lock_guard lock(gopy_cb_mutex); + int64_t h = gopy_cb_next++; + gopy_cb_registry[h] = std::move(fn); + return h; +} + +static void gopy_cb_unregister(int64_t h) { + std::lock_guard lock(gopy_cb_mutex); + gopy_cb_registry.erase(h); +} + +// Unregisters a callback's handle once the call it was passed to returns, +// even if that call raised: playing the same role gopyCallbackScope plays +// for cffi. +struct GopyCBGuard { + int64_t h; + ~GopyCBGuard() { gopy_cb_unregister(h); } +}; + +// Looks up the callable registered under h, or returns false if the call it +// was passed to has already returned (h was never valid, or was already +// unregistered). The caller must already hold the GIL (see +// callback_trampoline in nanobind_build.py for why it acquires that itself, +// rather than here). +static bool gopy_cb_lookup(int64_t h, nb::callable& out) { + std::lock_guard lock(gopy_cb_mutex); + auto it = gopy_cb_registry.find(h); + if (it == gopy_cb_registry.end()) { + return false; + } + out = it->second; + return true; +} + +@CALLBACK_TRAMPOLINES@ + +NB_MODULE(_@NAME@, m) { + // gen_slice.go always exports these 4 (under noAPIShim()) for the + // built-in byte slice, regardless of whether the package uses []byte; + // they exchange a raw pointer+length rather than a PyObject*, same as + // the cffi and pybind11 backends. + m.def("Slice_byte_from_bytes", [](nb::bytes b) -> int64_t { + return Slice_byte_from_bytes(const_cast(b.c_str()), (long long)b.size()); + }); + m.def("Slice_byte_to_bytes", [](int64_t handle) -> nb::bytes { + long long n = Slice_byte_to_bytes_len(handle); + if (n == 0) { + return nb::bytes("", 0); + } + void* ptr = Slice_byte_to_bytes_ptr(handle); + nb::bytes result(ptr, (size_t)n); + Slice_byte_free_ptr(ptr); + return result; + }); +@DEFS@ +} +''' + +mod = Module('_@NAME@', '@NAME@.cpp') +mod.add_include('"@NAME@_go.h"') +mod.add_function('GoPyInit', None, []) +mod.add_function('DecRef', None, [param('int64_t', 'handle')]) +mod.add_function('IncRef', None, [param('int64_t', 'handle')]) +mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) diff --git a/bind/noapi.go b/bind/noapi.go new file mode 100644 index 00000000..6c261a66 --- /dev/null +++ b/bind/noapi.go @@ -0,0 +1,128 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import "fmt" + +// A "no-API" backend generates exported Go functions that never touch the +// CPython C API, because their python-facing wrapper is produced outside the +// Go build: cffi loads a plain shared library at runtime; pybind11 and +// nanobind compile a C++ file against that same library. This file holds +// what they share about the shape of that library -- backend detection, +// error reporting (gopySetError, defined in cffi.go's preamble and used by +// all of them), and how a complex64/128 value crosses the boundary -- +// leaving each backend's own file for what only it needs (cffi.go, +// pybind11.go, nanobind.go). + +func (g *pyGen) isCFFI() bool { + return g.cfg.Backend == BackendCFFI +} + +func (g *pyGen) isPyBind11() bool { + return g.cfg.Backend == BackendPyBind11 +} + +func (g *pyGen) isNanobind() bool { + return g.cfg.Backend == BackendNanobind +} + +// isCXXShim reports whether the shim's consumer is a C++ module compiled +// against it (pybind11 or nanobind), which also means callbacks cross as +// registry handles (see pybind11_callback.go). +func (g *pyGen) isCXXShim() bool { + return g.isPyBind11() || g.isNanobind() +} + +// noAPIShim reports whether the exported Go functions must avoid the +// CPython C API (see the file comment above). +func (g *pyGen) noAPIShim() bool { + return g.isCFFI() || g.isCXXShim() +} + +// goSetError returns Go code that records an error for Python to raise. +// kind is the name of a Python builtin exception, msg a Go string expression. +func (g *pyGen) goSetError(kind, msg string) string { + return "gopySetError(\"" + kind + "\", " + msg + ")\n" +} + +func isComplexSym(sym *symbol) bool { + return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") +} + +// A complex64/complex128 value has no single C type that cffi or a C++ +// module can declare (cgo's is _Complex), and cgo won't export a struct, so +// under any of them it crosses as two floats: as two parameters (_re, _im), +// and as a result in cgo's two-value return, which it exports as a plain C +// struct {r0; r1;}. The methods below say how a value of a given symbol +// crosses, so that the generators only differ from the default backend here. + +// isComplexShim reports whether sym crosses as two floats, under either +// no-API backend (cffi, pybind11 or nanobind). +func (g *pyGen) isComplexShim(sym *symbol) bool { + return g.noAPIShim() && isComplexSym(sym) +} + +// cffiComplexFloat returns the cgo and the Go float type of the parts of a +// complex64 or complex128 symbol. +func cffiComplexFloat(sym *symbol) (cfloat, gofloat string) { + if sym.goname == "complex64" { + return "C.float", "float32" + } + return "C.double", "float64" +} + +// cgoParam returns the declaration of the parameter of an exported function +// that carries a value of sym. +func (g *pyGen) cgoParam(name string, sym *symbol) string { + if g.isComplexShim(sym) { + cf, _ := cffiComplexFloat(sym) + return fmt.Sprintf("%[1]s_re %[2]s, %[1]s_im %[2]s", name, cf) + } + return name + " " + sym.cgoname +} + +// cgoResult returns the result type of an exported function that returns a +// value of sym. +func (g *pyGen) cgoResult(sym *symbol) string { + if g.isComplexShim(sym) { + cf, _ := cffiComplexFloat(sym) + return "(" + cf + ", " + cf + ")" + } + return sym.cgoname +} + +// cpyName returns the type that build.py records for a value of sym. +// wrapper in cffi_build.py, pybind11_build.py and nanobind_build.py expands +// complex64/128. +func (g *pyGen) cpyName(sym *symbol) string { + if g.isComplexShim(sym) { + return sym.goname + } + return sym.cpyname +} + +// goToCgo returns the Go expression that converts expr, a value of sym, to +// what an exported function returns. +func (g *pyGen) goToCgo(sym *symbol, expr string) string { + switch { + case g.isComplexShim(sym): + return sym.goname + "GoToPyCFFI(" + expr + ")" + case sym.go2py != "": + return sym.go2py + "(" + expr + ")" + sym.go2pyParenEx + } + return expr +} + +// cgoToGo returns the Go expression that converts the parameter name, as +// declared by cgoParam, to a value of sym. +func (g *pyGen) cgoToGo(sym *symbol, name string) string { + switch { + case g.isComplexShim(sym): + return sym.goname + "PyToGoCFFI(" + name + "_re, " + name + "_im)" + case sym.py2go != "": + return sym.py2go + "(" + name + ")" + sym.py2goParenEx + } + return name +} diff --git a/bind/pybind11.go b/bind/pybind11.go new file mode 100644 index 00000000..9e5c217e --- /dev/null +++ b/bind/pybind11.go @@ -0,0 +1,29 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" +) + +// The pybind11 backend (GOPY_BACKEND=pybind11) shares its cgo shim with cffi +// (see noAPIShim in cffi.go): the same plain-C exported functions, the same +// gopySetError/GopyTakeError error channel, the same byte-slice and callback +// conventions. What differs is the consumer: instead of a pure-Python module +// that loads the shim at runtime with ctypes, pybind11Build.py writes a C++ +// file that #includes the shim's own header and compiles against it directly, +// so (unlike cffi_build.py) it needs no runtime description of the C types -- +// the C++ compiler gets them from the header, the same way pybindgen's +// generated .c file does. + +//go:embed pybind11_build.py +var pybind11BuildPy string + +// cxxArgHelpers is the argument conversion code shared by the pybind11 and +// nanobind backends' generated .cpp, which each build.py has in place of +// @ARG_HELPERS@. +// +//go:embed cxx_args.inc +var cxxArgHelpers string diff --git a/bind/pybind11_build.py b/bind/pybind11_build.py new file mode 100644 index 00000000..8cb2e6b0 --- /dev/null +++ b/bind/pybind11_build.py @@ -0,0 +1,356 @@ +# python build stub for package @NAME@ (pybind11 backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes +# @NAME@.cpp: a pybind11 module that #includes @NAME@_go.h (cgo's own header) +# and calls the shim functions directly, so -- unlike the cffi backend -- +# their C types come from the C++ compiler, not from parsing the header here. + + +def retval(ctype, *a, **kw): + return ctype + + +def param(ctype, name, *a, **kw): + return (ctype, name) + + +class Module(object): + def __init__(self, name, cpp_name): + self.name = name # the compiled extension's import name, e.g. "_hi" + self.cpp_name = cpp_name # source file to write, e.g. "hi.cpp" + self.header = None + self.funcs = [] + + def add_include(self, inc): + self.header = inc.strip('"') + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append((name, ret, params)) + + def generate(self): + import os + + here = os.path.dirname(os.path.abspath(__file__)) + # a callback's C++ trampoline is shared by every callable of the same + # shape (see pybind11_callback.go), keyed and numbered here in the + # same first-seen order Go numbered them in, so "gopy_cb_" means + # the same thing on both sides without the two ever exchanging it. + callback_kinds = {} + defs = [d for d in (wrapper(name, ret, params, callback_kinds) for name, ret, params in self.funcs) if d] + trampolines = "\n".join(callback_trampoline(ctype, i) for ctype, i in callback_kinds.items()) + cpp = ( + MODULE_TEMPLATE.replace("@HEADER@", self.header) + .replace("@CALLBACK_TRAMPOLINES@", trampolines) + .replace("@DEFS@", "\n".join(defs)) + ) + with open(os.path.join(here, self.cpp_name), "w") as f: + f.write(cpp) + + +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.add_function(name, retval, params) + + +add_checked_string_function = add_checked_function + + +# How wrapper converts an argument of each C type: as pybindgen's generated +# code would, for the same exception on a wrongly-typed argument under every +# backend (see the gopy_arg_* helpers in cxx_args.inc). %(o)s is the +# argument's PyObject*, %(n)d its 1-based position. Identical in +# pybind11_build.py and nanobind_build.py. +ARG_CONV = { + "int64_t": "gopy_arg_L(%(o)s)", + "long long": "gopy_arg_L(%(o)s)", + "uint64_t": "gopy_arg_K(%(o)s, %(n)d)", + "int": "gopy_arg_i(%(o)s)", + "int32_t": "gopy_arg_i(%(o)s)", + "uint32_t": "gopy_arg_I(%(o)s)", + "unsigned int": "gopy_arg_I(%(o)s)", + "int16_t": "(int16_t)gopy_arg_i_max(%(o)s, 0x7fff)", + "uint16_t": "(uint16_t)gopy_arg_i_max(%(o)s, 0xffff)", + "int8_t": "(int8_t)gopy_arg_i_max(%(o)s, 0x7f)", + "uint8_t": "(uint8_t)gopy_arg_i_max(%(o)s, 0xff)", + "double": "gopy_arg_d(%(o)s)", + "float": "(float)gopy_arg_d(%(o)s)", + "bool": "gopy_arg_bool(%(o)s)", + "char*": "const_cast(gopy_arg_str(%(o)s, %(n)d))", +} + + +def wrapper(name, ret, params, callback_kinds): + """Returns the m.def(...) call binding name, or "" if its signature + isn't supported yet (a raw PyObject*): the .cpp simply never binds it, so + calling it from python raises AttributeError instead of + NotImplementedError -- close enough for a function nothing in gopy's own + generated wrapper calls unconditionally. callback_kinds is shared across + every call from Module.generate, one entry per distinct callback shape + seen so far (see there and callback_trampoline). + """ + if ret == "PyObject*" or any(p[0] == "PyObject*" for p in params): + return "" + args = [] + setup = [] + call_args = [] + for i, (ctype, pname) in enumerate(params): + if ctype in ARG_CONV: + # converted up front, in order, while the GIL is still held + args.append("py::handle " + pname) + setup.append( + "auto _a_%s = %s;" % (pname, ARG_CONV[ctype] % {"o": pname + ".ptr()", "n": i + 1}) + ) + call_args.append("_a_" + pname) + elif ctype in ("complex64", "complex128"): + cxxfloat = "float" if ctype == "complex64" else "double" + args.append("std::complex<%s> %s" % (cxxfloat, pname)) + call_args.append("%s.real(), %s.imag()" % (pname, pname)) + elif ctype.startswith("callback:"): + i = callback_kinds.setdefault(ctype, len(callback_kinds)) + args.append("py::handle " + pname) + setup.append( + "gopy_arg_callable(%s.ptr(), %d);\n" + " int64_t _h_%s = gopy_cb_register(py::reinterpret_borrow(%s));\n" + " GopyCBGuard _g_%s{_h_%s};" % (pname, i + 1, pname, pname, pname, pname) + ) + call_args.append("_h_%s" % pname) + else: + args.append(ctype + " " + pname) + call_args.append(pname) + call = "%s(%s)" % (name, ", ".join(call_args)) + # Releasing the GIL only around the call itself (not the setup/result + # handling around it, which need it) matches what cffi gets for free + # from ctypes/cffi's own default behavior, and is what makes a callback + # arrive correctly rather than deadlock: Go may run it from a goroutine + # (see InGoroutine in _examples/callbacks) while this call's own thread + # blocks waiting for that goroutine, so it must not be left holding the + # only GIL there is. + if ret is None: + call = "[&]{ py::gil_scoped_release _rel; %s; }()" % call + else: + call = "[&]{ py::gil_scoped_release _rel; return %s; }()" % call + if ret is None: + body, cpptype = "%s;\n _check();" % call, "void" + elif ret == "char*": + body = ( + "char* _r = %s;\n" + " std::string _s(_r ? _r : \"\");\n" + " free(_r);\n" + " _check();\n" + " return _s;" % call + ) + cpptype = "std::string" + elif ret == "bool": + body = "auto _r = %s;\n _check();\n return _r != 0;" % call + cpptype = "bool" + elif ret in ("complex64", "complex128"): + cxxfloat = "float" if ret == "complex64" else "double" + cpptype = "std::complex<%s>" % cxxfloat + body = ( + "auto _r = %s;\n" + " _check();\n" + " return %s(_r.r0, _r.r1);" % (call, cpptype) + ) + else: + body = "auto _r = %s;\n _check();\n return _r;" % call + cpptype = ret + if setup: + body = "\n ".join(setup) + "\n " + body + return ' m.def("%s", [](%s) -> %s {\n %s\n });' % ( + name, + ", ".join(args), + cpptype, + body, + ) + + +def callback_trampoline(ctype, idx): + """Returns the static gopy_cb_ trampoline for the callback shape in + ctype ("callback:()", see cffiCallback + in cffi_callback.go): the Go closure for every callable of this shape + calls gopy_cb_, passing its own registry handle as the first + argument (see pybind11CallbackLit in pybind11_callback.go). + """ + ret, _, rest = ctype[len("callback:") :].partition("(") + ctypes_ = [t for t in rest[:-1].split(",") if t] + names = ["a%d" % i for i in range(len(ctypes_))] + + def cxxparam(t): + return "unsigned char" if t == "bool" else t + + params = "".join(", %s %s" % (cxxparam(t), n) for t, n in zip(ctypes_, names)) + call_args = [] + for t, n in zip(ctypes_, names): + if t == "char*": + call_args.append("%s ? py::str(%s) : py::str()" % (n, n)) + elif t == "bool": + call_args.append("py::bool_(%s != 0)" % n) + else: + call_args.append(n) + call = "fn(%s)" % ", ".join(call_args) + cxxret = "void" if ret == "void" else cxxparam(ret) + zero = "" if ret == "void" else " 0" + if ret == "void": + body = "%s;" % call + elif ret == "bool": + body = "return %s.cast() ? 1 : 0;" % call + else: + body = "return %s.cast<%s>();" % (call, ret) + # A raised exception must not reach the extern "C" boundary as a C++ + # exception: unwinding through Go's compiled call frames is undefined + # behavior (a hard crash in practice). Printing it and returning the + # zero value instead matches what cffi's ffi.callback does by default. + body = ( + "try {\n" + " %s\n" + " } catch (py::error_already_set& e) {\n" + " e.restore();\n" + " PyErr_Print();\n" + " return%s;\n" + " }" % (body, zero) + ) + return ( + # gil must be declared (and so acquired) before fn: C++ destroys + # locals in reverse declaration order, and fn (a py::function) needs + # the GIL held for its own destructor -- declared the other way + # around, gil would release it first, and fn would decref without it. + 'extern "C" %s gopy_cb_%d(int64_t h%s) {\n' + " py::gil_scoped_acquire gil;\n" + " py::function fn;\n" + " if (!gopy_cb_lookup(h, fn)) {\n" + " return%s;\n" + " }\n" + " %s\n" + "}" % (cxxret, idx, params, zero, body) + ) + + +MODULE_TEMPLATE = '''// python bindings for package @NAME@ using pybind11. +// File is generated by gopy version @VERSION@. Do not edit. +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +extern "C" { +#include "@HEADER@" +} + +// Hands the python error already set back to python, from C++. +[[noreturn]] static void gopy_raise() { + throw py::error_already_set(); +} + +@ARG_HELPERS@ + +// Raises the exception, if any, that the last Go call on this thread +// recorded (see gopySetError/GopyTakeError, shared with the cffi backend). +static inline void _check() { + char* e = GopyTakeError(); + if (!e) { + return; + } + std::string s(e); + free(e); + auto sep = s.find(':'); + std::string kind = sep == std::string::npos ? s : s.substr(0, sep); + std::string msg = sep == std::string::npos ? "" : s.substr(sep + 1); + PyObject* exc = PyExc_RuntimeError; + if (kind == "ValueError") exc = PyExc_ValueError; + else if (kind == "TypeError") exc = PyExc_TypeError; + else if (kind == "KeyError") exc = PyExc_KeyError; + else if (kind == "IndexError") exc = PyExc_IndexError; + else if (kind == "AttributeError") exc = PyExc_AttributeError; + PyErr_SetString(exc, msg.c_str()); + throw py::error_already_set(); +} + +// A python callable passed as a func-typed argument is registered here for +// the duration of the call it was passed to (see pybind11_callback.go for +// why a registry rather than one C function pointer per callable), and the +// trampolines below (one per callback shape, see callback_trampoline in +// pybind11_build.py) look it up by handle each time Go calls back in. +static std::mutex gopy_cb_mutex; +static std::unordered_map gopy_cb_registry; +static int64_t gopy_cb_next = 1; + +static int64_t gopy_cb_register(py::function fn) { + std::lock_guard lock(gopy_cb_mutex); + int64_t h = gopy_cb_next++; + gopy_cb_registry[h] = std::move(fn); + return h; +} + +static void gopy_cb_unregister(int64_t h) { + std::lock_guard lock(gopy_cb_mutex); + gopy_cb_registry.erase(h); +} + +// Unregisters a callback's handle once the call it was passed to returns, +// even if that call raised: playing the same role gopyCallbackScope plays +// for cffi. +struct GopyCBGuard { + int64_t h; + ~GopyCBGuard() { gopy_cb_unregister(h); } +}; + +// Looks up the callable registered under h, or returns false if the call it +// was passed to has already returned (h was never valid, or was already +// unregistered). The caller must already hold the GIL (see +// callback_trampoline in pybind11_build.py for why it acquires that itself, +// rather than here). +static bool gopy_cb_lookup(int64_t h, py::function& out) { + std::lock_guard lock(gopy_cb_mutex); + auto it = gopy_cb_registry.find(h); + if (it == gopy_cb_registry.end()) { + return false; + } + out = it->second; + return true; +} + +@CALLBACK_TRAMPOLINES@ + +PYBIND11_MODULE(_@NAME@, m) { + // gen_slice.go always exports these 4 (under noAPIShim()) for the + // built-in byte slice, regardless of whether the package uses []byte; + // they exchange a raw pointer+length rather than a PyObject*, same as + // the cffi backend, but bound directly here rather than by name-sniffing + // the header (cffi_build.py's BYTES_FUNCS) since nothing here needs to. + m.def("Slice_byte_from_bytes", [](py::bytes b) -> int64_t { + std::string s = b; + return Slice_byte_from_bytes(const_cast(s.data()), (long long)s.size()); + }); + m.def("Slice_byte_to_bytes", [](int64_t handle) -> py::bytes { + long long n = Slice_byte_to_bytes_len(handle); + if (n == 0) { + return py::bytes("", 0); + } + void* ptr = Slice_byte_to_bytes_ptr(handle); + py::bytes result(static_cast(ptr), (size_t)n); + Slice_byte_free_ptr(ptr); + return result; + }); +@DEFS@ +} +''' + +mod = Module('_@NAME@', '@NAME@.cpp') +mod.add_include('"@NAME@_go.h"') +mod.add_function('GoPyInit', None, []) +mod.add_function('DecRef', None, [param('int64_t', 'handle')]) +mod.add_function('IncRef', None, [param('int64_t', 'handle')]) +mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) diff --git a/bind/pybind11_callback.go b/bind/pybind11_callback.go new file mode 100644 index 00000000..01327552 --- /dev/null +++ b/bind/pybind11_callback.go @@ -0,0 +1,90 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "bytes" + "fmt" + "strings" +) + +// A Python callable passed to Go as a func-typed argument crosses the +// pybind11 boundary as an int64 handle, not a raw C function pointer: unlike +// cffi (whose ffi.callback creates one real function pointer per Python +// callable, via a libffi closure allocated at runtime), pybind11 has no way +// to synthesize new C-ABI function pointers at runtime, so instead one +// static C++ trampoline exists per callback signature, shared by every +// callable of that shape, and the actual py::function is looked up from a +// registry by handle each time Go calls back in (see MODULE_TEMPLATE and +// callback_trampoline in pybind11_build.py). Registering/unregistering the +// handle around the call (there, not here) is what makes a callback stop +// working once the python call it was passed to returns, playing the same +// role gopyCallbackScope plays for cffi. +// +// The parameter/result type rules are shared with cffi (cffiCallback, +// cffiCallbackParam, cffiCallbackResult in cffi_callback.go): the same Go +// types are supported, crossing as the same int64_t/uint64_t/double/bool/ +// char* vocabulary either way. +// +// Everything in this file serves the nanobind backend too (see isCXXShim), +// whose nanobind_build.py defines the same registry and trampolines. + +// pybind11TrampolinesKey stands in for the extern declarations in the cgo +// preamble, which are written before the callback types that need them are +// known. Unlike cffiTrampolinesKey, these are declarations only: the +// trampolines themselves are defined in the .cpp pybind11_build.py writes +// (callback_trampoline), not here -- see the buildCXXModule doc comment +// (cmd_build.go) for why Go and that .cpp can't be two separate libraries +// with a dependency in each direction. +const pybind11TrampolinesKey = "@@GOPY_PYBIND11_TRAMPOLINES@@" + +// splicePyBind11Trampolines writes the extern declarations into the cgo +// preamble, so the C compiler accepts calls to a function it never sees +// defined; the actual gopy_cb_N functions are resolved at the final link +// step in buildCXXModule, against the object code pybind11_build.py's +// generated .cpp compiles to. +func (g *pyGen) splicePyBind11Trampolines() { + if !g.isCXXShim() { + return + } + var c strings.Builder + for i, cb := range g.cbs { + params := []string{"int64_t h"} + for j, p := range cb.params { + params = append(params, fmt.Sprintf("%s a%d", cffiCType(p.ctype), j)) + } + ret := "void" + if cb.ret != nil { + ret = cffiCType(cb.ret.ctype) + } + fmt.Fprintf(&c, "extern %s gopy_cb_%d(%s);\n", ret, i, strings.Join(params, ", ")) + } + b := bytes.Replace(g.gofile.buf.Bytes(), []byte(pybind11TrampolinesKey), []byte(c.String()), 1) + g.gofile.buf = bytes.NewBuffer(b) +} + +// pybind11CallbackLit returns a Go func literal that calls the Python +// callable registered under the handle named anm. +func (g *pyGen) pybind11CallbackLit(cb *cffiCallback, anm string) string { + var decl, pre []string + args := []string{"C.int64_t(" + anm + ")"} + for _, p := range cb.params { + decl = append(decl, p.name+" "+p.gotyp) + if p.pre != "" { + pre = append(pre, p.pre) + } + args = append(args, p.conv) + } + call := fmt.Sprintf("C.gopy_cb_%d(%s)", g.cffiTrampoline(cb), strings.Join(args, ", ")) + result := "" + if cb.ret != nil { + result = " " + cb.ret.gotyp + if cb.ret.ctype == "bool" { + call += " != 0" + } + call = "return " + cb.ret.gotyp + "(" + call + ")" + } + return fmt.Sprintf("func(%s)%s {\n%s%s\n}", strings.Join(decl, ", "), result, strings.Join(pre, ""), call) +} diff --git a/cmd_build.go b/cmd_build.go index 7e38997f..8d466349 100644 --- a/cmd_build.go +++ b/cmd_build.go @@ -31,7 +31,7 @@ build generates and compiles (C)Python language bindings for Go package(s). ex: $ gopy build [options] [other-go-package...] $ gopy build github.com/go-python/gopy/_examples/hi -`, +` + backendHelp, Flag: *flag.NewFlagSet("gopy-build", flag.ExitOnError), } @@ -115,15 +115,25 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { os.Remove(cfg.Name + ".c") // may fail, we don't care - fmt.Printf("goimports -w %v\n", cfg.Name+".go") - cmd := exec.Command("goimports", "-w", cfg.Name+".go") - cmdout, err = cmd.CombinedOutput() - if err != nil { - fmt.Printf("cmd had error: %v output:\no%v\n", err, string(cmdout)) + if err := runCmd(nil, "goimports", "-w", cfg.Name+".go"); err != nil { return err } + if cfg.Backend == bind.BackendCFFI { + return buildCFFI(cfg, buildname+libExt) + } + pycfg, err := bind.GetPythonConfig(cfg.VM) + if err != nil { + return err + } + + switch cfg.Backend { + case bind.BackendPyBind11: + return buildPyBind11(cfg, pycfg) + case bind.BackendNanobind: + return buildNanobind(cfg, pycfg) + } if mode == bind.ModeExe { of, err := os.Create(buildname + ".h") // overwrite existing @@ -131,7 +141,7 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { of.Close() fmt.Printf("%v build.py # will fail, but needed to generate .c file\n", cfg.VM) - cmd = exec.Command(cfg.VM, "build.py") + cmd := exec.Command(cfg.VM, "build.py") cmd.Run() // will fail, we don't care about errors args := []string{"build", "-mod=mod", "-buildmode=c-shared"} @@ -183,23 +193,7 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { // build the go shared library upfront to generate the header // needed by our generated cpython code - firstArgs := []string{"build", "-mod=mod", "-buildmode=c-shared"} - if cfg.BuildTags != "" { - firstArgs = append(firstArgs, "-tags", cfg.BuildTags) - } - if !cfg.Symbols { - // These flags will omit the various symbol tables, thereby - // reducing the final size of the binary. From https://golang.org/cmd/link/ - // -s Omit the symbol table and debug information - // -w Omit the DWARF symbol table - firstArgs = append(firstArgs, "-ldflags=-s -w") - } - firstArgs = append(firstArgs, "-o", buildLib, ".") - fmt.Printf("go %v\n", strings.Join(firstArgs, " ")) - cmd = exec.Command("go", firstArgs...) - cmdout, err = cmd.CombinedOutput() - if err != nil { - fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout)) + if err := runCmd(nil, "go", goBuildArgs(cfg, "c-shared", buildLib)...); err != nil { return err } // we don't need this initial lib because we are going to relink @@ -211,14 +205,7 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { // via RTLD_GLOBAL interposition corrupt each other's GC state (#370). // This applies only to the second build, which is where PyInit__ // exists and where the exported-symbols list is valid. - finalArgs := []string{"build", "-mod=mod", "-buildmode=c-shared"} - if cfg.BuildTags != "" { - finalArgs = append(finalArgs, "-tags", cfg.BuildTags) - } - var finalLdFlags []string - if !cfg.Symbols { - finalLdFlags = append(finalLdFlags, "-s", "-w") - } + var exportLdFlags []string switch runtime.GOOS { case "darwin": ef, ferr := os.CreateTemp("", "gopy-exports-*.txt") @@ -226,7 +213,7 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { fmt.Fprintf(ef, "_PyInit__%s\n", cfg.Name) ef.Close() defer os.Remove(ef.Name()) - finalLdFlags = append(finalLdFlags, "-extldflags=-Wl,-exported_symbols_list,"+ef.Name()) + exportLdFlags = append(exportLdFlags, "-extldflags=-Wl,-exported_symbols_list,"+ef.Name()) } case "linux": ef, ferr := os.CreateTemp("", "gopy-exports-*.map") @@ -234,22 +221,12 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { fmt.Fprintf(ef, "{ global: PyInit__%s; local: *; };\n", cfg.Name) ef.Close() defer os.Remove(ef.Name()) - finalLdFlags = append(finalLdFlags, "-extldflags=-Wl,--version-script="+ef.Name()) + exportLdFlags = append(exportLdFlags, "-extldflags=-Wl,--version-script="+ef.Name()) } } - if len(finalLdFlags) > 0 { - finalArgs = append(finalArgs, "-ldflags="+strings.Join(finalLdFlags, " ")) - } - finalArgs = append(finalArgs, "-o", modlib, ".") - // args is still used below for the CGO env build; point it at finalArgs. - args := finalArgs // generate c code - fmt.Printf("%v build.py\n", cfg.VM) - cmd = exec.Command(cfg.VM, "build.py") - cmdout, err = cmd.CombinedOutput() - if err != nil { - fmt.Printf("cmd had error: %v output:\no%v\n", err, string(cmdout)) + if err := runCmd(nil, cfg.VM, "build.py"); err != nil { return err } @@ -321,15 +298,103 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { fmt.Println(ldflagsEnv) // build extension with go + c - fmt.Printf("go %v\n", strings.Join(args, " ")) - cmd = exec.Command("go", args...) - cmd.Env = env - cmdout, err = cmd.CombinedOutput() - if err != nil { - fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout)) + if err := runCmd(env, "go", goBuildArgs(cfg, "c-shared", modlib, exportLdFlags...)...); err != nil { return err } } return err } + +// buildCFFI builds the cgo shim as a plain shared library, and then runs +// build.py to write the cffi module that loads it. The current directory +// is the output directory. +func buildCFFI(cfg *BuildCfg, buildLib string) error { + if err := runCmd(nil, "go", goBuildArgs(cfg, "c-shared", buildLib)...); err != nil { + return err + } + return runCmd(nil, cfg.VM, "build.py") +} + +// buildPyBind11 builds the pybind11 backend's module (see buildCXXModule). +func buildPyBind11(cfg *BuildCfg, pycfg bind.PyConfig) error { + cmdout, err := exec.Command(cfg.VM, "-m", "pybind11", "--includes").CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n(is pybind11 installed? pip install pybind11)\n", err, string(cmdout)) + return err + } + return buildCXXModule(cfg, pycfg, strings.Fields(strings.TrimSpace(string(cmdout))), nil) +} + +// buildNanobind builds the nanobind backend's module (see buildCXXModule). +// Unlike pybind11, nanobind isn't header-only: its own runtime (libnanobind) +// ships as source, meant to be compiled into each extension alongside the +// extension's own code, which nb_combined.cpp does in one translation unit. +func buildNanobind(cfg *BuildCfg, pycfg bind.PyConfig) error { + cmdout, err := exec.Command(cfg.VM, "-c", + "import nanobind; print(nanobind.include_dir()); print(nanobind.source_dir())").CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n(is nanobind installed? pip install nanobind)\n", err, string(cmdout)) + return err + } + dirs := strings.Split(strings.TrimSpace(string(cmdout)), "\n") + if len(dirs) != 2 { + return fmt.Errorf("gopy: unexpected output locating nanobind: %q", string(cmdout)) + } + incdir, srcdir := strings.TrimSpace(dirs[0]), strings.TrimSpace(dirs[1]) + // robin_map is a dependency nanobind vendors next to its own headers. + robinmap := filepath.Join(filepath.Dir(incdir), "ext", "robin_map", "include") + flags := append([]string{"-I" + incdir, "-I" + robinmap}, bind.NanobindCXXFlags...) + return buildCXXModule(cfg, pycfg, flags, []string{filepath.Join(srcdir, "nb_combined.cpp")}) +} + +// buildCXXModule builds the cgo shim as a static archive, runs build.py to +// write a C++ module wrapping it (pybind11 or nanobind), and compiles+links +// that with a C++ compiler (see bind.CXXArgs, and bind/cxxbuild.go for why +// a static archive), passing it cxxflags (the C++ library's include +// directories, and any flags of its own) and, besides the generated .cpp, +// the C++ library's own sources, if any. The current directory is the +// output directory. The Makefile gopy gen writes for these backends runs +// the same steps. +func buildCXXModule(cfg *BuildCfg, pycfg bind.PyConfig, cxxflags, srcs []string) error { + if err := runCmd(nil, "go", goBuildArgs(cfg, "c-archive", bind.CXXArchive(cfg.Name))...); err != nil { + return err + } + if err := runCmd(nil, cfg.VM, "build.py"); err != nil { + return err + } + cxxArgs := bind.CXXArgs(cfg.Name, bind.ExtModuleName(cfg.Name, libExt, pycfg), pycfg, cxxflags, srcs) + return runCmd(nil, bind.CXX(), cxxArgs...) +} + +// goBuildArgs returns the go build arguments that build the package in the +// current directory into out with the given -buildmode, with cfg's build +// tags and, unless cfg.Symbols is set, without symbol tables (-s omits the +// symbol table and debug information, -w the DWARF symbol table; see +// https://golang.org/cmd/link/). ldflags are any further linker flags. +func goBuildArgs(cfg *BuildCfg, buildmode, out string, ldflags ...string) []string { + args := []string{"build", "-mod=mod", "-buildmode=" + buildmode} + if cfg.BuildTags != "" { + args = append(args, "-tags", cfg.BuildTags) + } + if !cfg.Symbols { + ldflags = append([]string{"-s", "-w"}, ldflags...) + } + if len(ldflags) > 0 { + args = append(args, "-ldflags="+strings.Join(ldflags, " ")) + } + return append(args, "-o", out, ".") +} + +// runCmd runs name with args, in env if it isn't nil, printing the command +// line first, and its output if it fails. +func runCmd(env []string, name string, args ...string) error { + fmt.Printf("%s %s\n", name, strings.Join(args, " ")) + cmd := exec.Command(name, args...) + cmd.Env = env + out, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n", err, string(out)) + } + return err +} diff --git a/cmd_gen.go b/cmd_gen.go index 8b767354..7e6687a8 100644 --- a/cmd_gen.go +++ b/cmd_gen.go @@ -13,6 +13,16 @@ import ( "github.com/gonuts/flag" ) +// backendHelp ends the help text of the commands that GOPY_BACKEND applies to. +const backendHelp = ` +backends: + GOPY_BACKEND chooses the tool that binds Go to python: pybindgen (the + default), capi, cffi, pybind11 or nanobind. capi needs nothing beyond python + itself; the others need their own python package (pip install ), and + pybind11 and nanobind also a C++ compiler. + $ GOPY_BACKEND=capi gopy build github.com/go-python/gopy/_examples/hi +` + func gopyMakeCmdGen() *commander.Command { cmd := &commander.Command{ Run: gopyRunCmdGen, @@ -24,7 +34,7 @@ gen generates (C)Python language bindings for Go package(s). ex: $ gopy gen [options] [other-go-package...] $ gopy gen github.com/go-python/gopy/_examples/hi -`, +` + backendHelp, Flag: *flag.NewFlagSet("gopy-gen", flag.ExitOnError), } diff --git a/cmd_pkg.go b/cmd_pkg.go index 9891907c..d073ef64 100644 --- a/cmd_pkg.go +++ b/cmd_pkg.go @@ -34,7 +34,7 @@ When including multiple packages, list in order of increasing dependency, and us ex: $ gopy pkg [options] [other-go-package...] $ gopy pkg github.com/go-python/gopy/_examples/hi -`, +` + backendHelp, Flag: *flag.NewFlagSet("gopy-pkg", flag.ExitOnError), } diff --git a/gen.go b/gen.go index 549ee2ad..8106db3b 100644 --- a/gen.go +++ b/gen.go @@ -63,6 +63,16 @@ func genOutDir(odir string) (string, error) { // mode = gen, build, pkg, exe func genPkg(mode bind.BuildMode, cfg *BuildCfg) error { var err error + if cfg.Backend, err = bind.BackendFromEnv(); err != nil { + return err + } + if (cfg.Backend == bind.BackendCFFI || cfg.Backend == bind.BackendPyBind11 || cfg.Backend == bind.BackendNanobind) && mode == bind.ModeExe { + // exe mode embeds the Python interpreter into the Go binary via the + // CPython C API (see goExePreambleC/Go in bind/gen.go), unrelated to + // how the bindings themselves are generated; none of these backends + // supports it. + return fmt.Errorf("gopy: %s=%s does not support gopy exe", bind.BackendEnvVar, cfg.Backend) + } cfg.OutputDir, err = genOutDir(cfg.OutputDir) if err != nil { return err @@ -78,7 +88,7 @@ func genPkg(mode bind.BuildMode, cfg *BuildCfg) error { if err != nil { return err } - err = bind.GenPyBind(mode, libExt, extraGccArgs, pyvers, cfg.DynamicLinking, &cfg.BindCfg) + err = bind.GenPyBind(mode, libExt, pyvers, cfg.DynamicLinking, &cfg.BindCfg) if err != nil { log.Println(err) } diff --git a/main_darwin.go b/main_darwin.go index 5edc9205..64fe8076 100644 --- a/main_darwin.go +++ b/main_darwin.go @@ -7,8 +7,5 @@ package main -const ( - // libExt = ".dylib" // theoretically should be this but python only recognizes .so - libExt = ".so" - extraGccArgs = "-dynamiclib" -) +// libExt = ".dylib" // theoretically should be this but python only recognizes .so +const libExt = ".so" diff --git a/main_test.go b/main_test.go index 0e0f7391..153a1475 100644 --- a/main_test.go +++ b/main_test.go @@ -51,6 +51,7 @@ var ( "_examples/pkgconflict": []string{"py3"}, "_examples/variadic": []string{"py3"}, "_examples/gilstring": []string{"py3"}, + "_examples/callbacks": []string{"py3"}, } testEnvironment = os.Environ() @@ -392,15 +393,55 @@ OK }) } -func TestBindSimple(t *testing.T) { +func TestBindCallbacks(t *testing.T) { // t.Parallel() - path := "_examples/simple" + path := "_examples/callbacks" testPkg(t, pkg{ path: path, lang: features[path], cmd: "build", extras: nil, - want: []byte(`doc(pkg): + want: []byte(`--- Each: int and string arguments +each: 0 item-0 +each: 1 item-1 +each: 2 item-2 +--- Mixed: bool, float and uint8 arguments +mixed: True 1.5 200 +mixed: False -2.25 7 +--- Twice: no arguments +twice: 2 +--- Counter.Visit: a Go struct arrives as a handle +visit: 1 1 +visit: 2 2 +counter: 2 +--- Describe: an interface{} arrives as a string +describe: 'a string' +describe: '1.5s' +--- Count: a bool result +count: 4 +--- Sum: an int result +sum: 30 +--- Widest: a uint result +widest: 30 +--- Apply: a float result +apply: 3.0 +--- Counter.Check: a handle argument and a bool result +check: True +--- a bound method +box: [(0, 'item-0'), (1, 'item-1')] +--- called from another goroutine +goroutine: 7 +--- an exception in a callback is reported, and Go carries on +calls: 3 reported: 3 +--- a callback Go keeps and calls after the call it was passed to returned +kept: exit code 0 +OK +`), + }) +} + +// simpleWant is _examples/simple/test.py's expected output. +var simpleWant = []byte(`doc(pkg): '\nsimple is a simple package.\n\n' pkg.Func()... fct = pkg.Func... @@ -411,7 +452,36 @@ pkg.Bool(False)= False pkg.Comp64Add((3+4j), (2+5j)) = (5+9j) pkg.Comp128Add((3+4j), (2+5j)) = (5+9j) OK -`), +`) + +func TestBindSimple(t *testing.T) { + // t.Parallel() + path := "_examples/simple" + testPkg(t, pkg{ + path: path, + lang: features[path], + cmd: "build", + extras: nil, + want: simpleWant, + }) +} + +// TestMakefile builds _examples/simple with the Makefile gopy gen writes, +// rather than with gopy build, under the backend GOPY_BACKEND selects. +func TestMakefile(t *testing.T) { + if _, err := bind.BackendFromEnv(); err != nil { + t.Fatal(err) + } + if _, err := exec.LookPath("make"); err != nil { + t.Skip("make not found") + } + path := "_examples/simple" + testPkg(t, pkg{ + path: path, + lang: features[path], + cmd: "gen", + make: true, + want: simpleWant, }) } @@ -1083,6 +1153,8 @@ type pkg struct { testdir string extras []string want []byte + // make runs the generated Makefile's build target after cmd (gen) + make bool } func testPkg(t *testing.T, table pkg) { @@ -1149,7 +1221,7 @@ func testPkgBackend(t *testing.T, pyvm string, table pkg) { // fmt.Printf("building in work dir: %s\n", workdir) fpath := "./" + table.path - if table.cmd != "build" { // non-build cases end up inside the working dir -- need a global import path + if table.cmd != "build" && table.cmd != "gen" { // non-build cases end up inside the working dir -- need a global import path fpath = filepath.Join(curPkgPath, table.path) } args := []string{table.cmd, "-vm=" + pyvm, "-output=" + genPkgDir, "-package-prefix", table.pkgprefix} @@ -1163,9 +1235,18 @@ func testPkgBackend(t *testing.T, pyvm string, table pkg) { t.Fatalf("[%s:%s]: error running gopy-build: %v\n", pyvm, table.path, err) } + if table.make { + fmt.Printf("running make build\n") + cmd := exec.Command("make", "build") + cmd.Dir = genPkgDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("[%s:%s]: error running make build: %v\n%s", pyvm, table.path, err, out) + } + } + // fmt.Printf("copying test.py\n") tstDir := genPkgDir - if table.cmd != "build" { + if table.cmd != "build" && table.cmd != "gen" { tstDir = filepath.Join(workdir, pkgNm) } if table.testdir != "" { diff --git a/main_unix.go b/main_unix.go index bd84a696..c7d9d2fc 100644 --- a/main_unix.go +++ b/main_unix.go @@ -7,7 +7,4 @@ package main -const ( - libExt = ".so" - extraGccArgs = "" -) +const libExt = ".so" diff --git a/main_windows.go b/main_windows.go index 9800b099..049b17c3 100644 --- a/main_windows.go +++ b/main_windows.go @@ -9,10 +9,7 @@ package main import "github.com/go-python/gopy/bind" -const ( - libExt = ".pyd" - extraGccArgs = "" -) +const libExt = ".pyd" func init() { bind.WindowsOS = true