From 936db2035f247ddc51468a583c878ffc6b66b5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Thu, 19 Nov 2026 22:10:42 -0500 Subject: [PATCH 1/2] A confined grant worker cannot start a process by clone3 (#28) The allow-list answers clone3 with ENOSYS, so the C library starts threads with clone, whose flags the filter checks for CLONE_THREAD. ADR 0002 records it as the one call outside the allow-list that does not kill. Co-Authored-By: Claude Fable 5.2 --- .../countersign-gateway/src/worker_sandbox.rs | 16 ++++--- .../tests/worker_sandbox.rs | 45 +++++++++++++++++++ ...ls-as-unsupported-in-the-worker-sandbox.md | 36 +++++++++++++++ 3 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 docs/adr/0002-clone3-fails-as-unsupported-in-the-worker-sandbox.md diff --git a/crates/countersign-gateway/src/worker_sandbox.rs b/crates/countersign-gateway/src/worker_sandbox.rs index 33c65ed..58697fb 100644 --- a/crates/countersign-gateway/src/worker_sandbox.rs +++ b/crates/countersign-gateway/src/worker_sandbox.rs @@ -184,8 +184,7 @@ const ALLOWED: &[libc::c_long] = &[ #[cfg(target_arch = "x86_64")] libc::SYS_epoll_wait, libc::SYS_eventfd2, - // Threads and their synchronisation. - libc::SYS_clone3, + // Threads' synchronisation. Starting one is `clone`, below. libc::SYS_set_robust_list, libc::SYS_rseq, libc::SYS_futex, @@ -261,6 +260,14 @@ fn allow_list() -> io::Result<()> { ), kill, ]); + // clone3 takes its flags in a structure, which a filter cannot read, so + // it cannot be allowed for threads alone. It is answered as a kernel + // without it answers, and the C library then starts the thread with clone. + // The one call outside the allow-list that does not kill (ADR 0002). + filter.extend([ + op(EQUAL, number(libc::SYS_clone3), 0, 1), + op(RETURN, libc::SECCOMP_RET_ERRNO | libc::ENOSYS as u32, 0, 0), + ]); for nr in ALLOWED { filter.extend([op(EQUAL, number(*nr), 0, 1), allow]); } @@ -285,9 +292,8 @@ fn allow_list() -> io::Result<()> { ]), )], ), - // A thread, where the host has no clone3: Docker's default profile - // answers clone3 with ENOSYS, and the C library falls back to clone. - // Unlike clone3's, clone's flags are an argument the filter can read. + // A thread. Unlike clone3's, clone's flags are an argument the filter + // can read. ( libc::SYS_clone, &[(DATA_ARG0, Test::Bit(libc::CLONE_THREAD as u32))], diff --git a/crates/countersign-gateway/tests/worker_sandbox.rs b/crates/countersign-gateway/tests/worker_sandbox.rs index 14eb88e..389c594 100644 --- a/crates/countersign-gateway/tests/worker_sandbox.rs +++ b/crates/countersign-gateway/tests/worker_sandbox.rs @@ -202,6 +202,7 @@ mod linux { "trace-me" => trace_me(), "raise" => raise(number(1) as libc::c_int), "fork" => fork(), + "clone3" => clone3(), "signal-on-io" => signal_on_io(number(1) as libc::pid_t), #[cfg(target_arch = "x86_64")] "x32" => x32(), @@ -322,6 +323,27 @@ mod linux { checked(pid) } + /// A second process by `clone3`, whose flags are in a structure that a + /// seccomp filter cannot read. It ends at once. + fn clone3() -> Result<(), i32> { + // SAFETY: an all-zero `clone_args` is valid: no flags, so a new process. + let mut args: libc::clone_args = unsafe { std::mem::zeroed() }; + args.exit_signal = libc::SIGCHLD as u64; + // SAFETY: `args` outlives the call, with its size. The new process + // runs on a copy of this stack and calls only `_exit`. + let pid = unsafe { + libc::syscall( + libc::SYS_clone3, + &raw const args, + size_of::(), + ) + }; + if pid == 0 { + unsafe { libc::_exit(0) }; + } + checked(pid) + } + /// Ask the kernel to signal `pid` whenever the control socket has input: /// a signal to another process that `kill` never sends. fn signal_on_io(pid: libc::pid_t) -> Result<(), i32> { @@ -702,6 +724,29 @@ mod linux { refused_only_when_confined(|_| "fork".into()); each_of_two(NO_CLONE3, |_| "fork".into(), Outcome::refused, "a refusal"); }), + ( + "starting_another_process_by_clone3_fails_as_unsupported", + || { + each_of_two( + UNCONFINED, + |_| "clone3".into(), + |o| *o == Outcome::GotThrough, + "success", + ); + // Not a kill: the C library asks for a thread by clone3 first, and + // takes this answer as its cue to use clone (ADR 0002). + for attacker in 0..2 { + let mut pair = [StandIn::start(CONFINED), StandIn::start(CONFINED)]; + assert_eq!( + pair[attacker].attempt("clone3"), + Outcome::Errno(libc::ENOSYS) + ); + // The stand-in lives on, and so does the other. + assert_eq!(pair[attacker].attempt("ping"), Outcome::GotThrough); + assert_eq!(pair[1 - attacker].attempt("ping"), Outcome::GotThrough); + } + }, + ), ( "having_the_kernel_signal_another_process_is_refused", || { diff --git a/docs/adr/0002-clone3-fails-as-unsupported-in-the-worker-sandbox.md b/docs/adr/0002-clone3-fails-as-unsupported-in-the-worker-sandbox.md new file mode 100644 index 0000000..20aae3f --- /dev/null +++ b/docs/adr/0002-clone3-fails-as-unsupported-in-the-worker-sandbox.md @@ -0,0 +1,36 @@ +--- +status: accepted +date: 2026-11-19 +supersedes: 0001, decision 5, for the `clone3` system call only +--- + +# `clone3` fails as unsupported in the worker sandbox + +A grant worker serves its connections on threads, and must not be able to start a second process. A second process inherits every base layer, but under the filter sandbox it has no process namespace around it: it would outlive the supervisor's kill of the grant worker and keep the stream descriptors it inherited. + +The seccomp allow-list can tell a thread from a process under `clone`, whose flags are an argument: it allows `clone` only with `CLONE_THREAD`. It cannot under `clone3`, which takes its flags in a structure, and a seccomp filter reads arguments but not the memory they point to. The first allow-list (#22) allowed `clone3` outright, because the C library starts threads with it. So a compromised grant worker could start a process wherever the kernel has `clone3` (#28). + +[ADR 0001](0001-filter-sandbox-for-hosts-without-namespaces.md), decision 5, leaves two answers, and neither works: allow `clone3`, and with it a new process; or leave it off the allow-list, which kills the grant worker when it starts its first thread. + +## Decision + +1. **The allow-list answers `clone3` with `ENOSYS`**, as a kernel older than `clone3` does. The C library takes that answer as its cue to start the thread with `clone`, which the allow-list checks for `CLONE_THREAD`. +2. **This is the one system call outside the allow-list that does not kill the grant worker.** For every other call, ADR 0001's decision 5 stands. Another exception needs its own ADR. +3. **Both worker sandboxes get it**, because it is part of the base layers. + +## Evidence + +- The containment test makes a raw `clone3` call without `CLONE_THREAD` from each of two concurrent stand-in grant workers. Before this decision the call started a process. Now it fails with `ENOSYS`, and the stand-in goes on serving its stream on a thread it started after confinement. Measured on glibc 2.39, arm64, kernel 6.17, on 2026-11-19; CI repeats it on x86-64 and arm64. +- Docker's default seccomp profile has answered `clone3` with `ENOSYS` since 20.10.10, for the same reason: its filter cannot read the flags either. So every C library that runs in a container already handles this answer, and a grant worker under Docker's default profile already ran this way before this decision. + +## Considered options + +- **Keep allowing `clone3`** and record why a second process is acceptable. Rejected: under the filter sandbox the second process escapes the supervisor's kill, which is how a grant is cut off, and holds the grant's streams. The base layers it inherits bound what it can do, not how long it lives. +- **Kill on `clone3`**, as decision 5 says. Rejected: the C library tries `clone3` first for every thread, so the grant worker would die serving its first connection. +- **Allow `clone3` and catch the new process elsewhere**, for example by killing the grant worker's process group, or with a PID limit. Rejected: the filter sandbox exists for hosts that offer an unprivileged supervisor no such control, and each of these is a second mechanism to keep correct where the filter needs one line. + +## Consequences + +- A compromised grant worker that tries `clone3` is not killed, and its grant is not fenced for it. It learns only that the call is unsupported, and its next forbidden call kills it. +- Code that calls `clone3` directly, without the C library's fallback, fails with `ENOSYS` in a grant worker instead of being killed. Nothing in the grant worker does today, and the runtime test would show it as a failed connection. +- A C library that stopped falling back to `clone` would leave the grant worker unable to start threads. The containment test and the runtime test, on both architectures, would fail first. From ea413ce28e191a56e6d8cf025a1fa8c41813178a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Thu, 19 Nov 2026 22:15:29 -0500 Subject: [PATCH 2/2] The clone3 test runs on a host without clone3, and the filter's doc names its exception (#28) Co-Authored-By: Claude Fable 5.2 --- .../countersign-gateway/src/worker_sandbox.rs | 3 +- .../tests/worker_sandbox.rs | 39 +++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/crates/countersign-gateway/src/worker_sandbox.rs b/crates/countersign-gateway/src/worker_sandbox.rs index 58697fb..2c9118b 100644 --- a/crates/countersign-gateway/src/worker_sandbox.rs +++ b/crates/countersign-gateway/src/worker_sandbox.rs @@ -231,7 +231,8 @@ enum Test<'a> { Bit(u32), } -/// The seccomp filter. A system call outside it kills the whole process. +/// The seccomp filter. A system call outside it kills the whole process, +/// bar `clone3`, which fails as unsupported (ADR 0002). fn allow_list() -> io::Result<()> { const LOAD: u16 = (libc::BPF_LD | libc::BPF_W | libc::BPF_ABS) as u16; const EQUAL: u16 = (libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K) as u16; diff --git a/crates/countersign-gateway/tests/worker_sandbox.rs b/crates/countersign-gateway/tests/worker_sandbox.rs index 389c594..5f223d5 100644 --- a/crates/countersign-gateway/tests/worker_sandbox.rs +++ b/crates/countersign-gateway/tests/worker_sandbox.rs @@ -44,8 +44,9 @@ mod linux { const LANDLOCK_REFUSED: &str = "landlock-refused"; /// A container profile older than Landlock answers its calls this way. const LANDLOCK_DENIED: &str = "landlock-denied"; - /// Docker's default profile answers `clone3` this way, and the C library - /// then starts threads with `clone`. + /// Docker's default profile answers `clone3` this way, ahead of the + /// allow-list, which gives the same answer (ADR 0002). So this setup + /// differs from a confined one only before confinement. const NO_CLONE3: &str = "no-clone3"; /// Stands in for a secret in the supervisor's or a grant worker's memory. @@ -727,24 +728,28 @@ mod linux { ( "starting_another_process_by_clone3_fails_as_unsupported", || { + // A host may answer clone3 this way itself, as Docker's default + // profile and a kernel before 5.3 do. There the call cannot be + // shown to get through unconfined. + if StandIn::start(UNCONFINED).attempt("clone3") == Outcome::Errno(libc::ENOSYS) { + println!(" this host has no clone3: the call fails unconfined too"); + } else { + each_of_two( + UNCONFINED, + |_| "clone3".into(), + |o| *o == Outcome::GotThrough, + "success", + ); + } + // Not a kill: the C library asks for a thread by clone3 first, and + // takes this answer as its cue to use clone (ADR 0002). The reply + // itself shows the stand-in lived to serve its stream. each_of_two( - UNCONFINED, + CONFINED, |_| "clone3".into(), - |o| *o == Outcome::GotThrough, - "success", + |o| *o == Outcome::Errno(libc::ENOSYS), + "ENOSYS", ); - // Not a kill: the C library asks for a thread by clone3 first, and - // takes this answer as its cue to use clone (ADR 0002). - for attacker in 0..2 { - let mut pair = [StandIn::start(CONFINED), StandIn::start(CONFINED)]; - assert_eq!( - pair[attacker].attempt("clone3"), - Outcome::Errno(libc::ENOSYS) - ); - // The stand-in lives on, and so does the other. - assert_eq!(pair[attacker].attempt("ping"), Outcome::GotThrough); - assert_eq!(pair[1 - attacker].attempt("ping"), Outcome::GotThrough); - } }, ), (