diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7c5c443..bd5068c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -33,6 +33,25 @@ jobs: - name: Name the system call when the worker sandbox's filter killed a grant worker if: failure() run: sudo dmesg | grep 'type=1326' | tail -n 20 || true + gateway-filter-sandbox: + # A container under Docker's default security profile refuses namespaces, + # as ECS Fargate does. The script runs the containment test and the + # runtime test under the filter sandbox there, and proves that the + # namespace sandbox fails at startup with the named error. + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - name: Run the Gateway under the filter sandbox where namespaces are refused + run: scripts/filter-sandbox-container.sh + - name: Name the system call when the worker sandbox's filter killed a grant worker + if: failure() + run: sudo dmesg | grep 'type=1326' | tail -n 20 || true curl: runs-on: ubuntu-24.04 timeout-minutes: 40 diff --git a/crates/countersign-gateway/src/runtime/config.rs b/crates/countersign-gateway/src/runtime/config.rs index ffd27dc..a73a6d9 100644 --- a/crates/countersign-gateway/src/runtime/config.rs +++ b/crates/countersign-gateway/src/runtime/config.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::fmt; use std::fs; use std::io; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; @@ -27,12 +28,45 @@ pub(crate) struct Config { #[serde(default)] pub denied_networks: Vec, pub shared: Option, + #[serde(default)] + pub worker_sandbox: WorkerSandbox, /// Conservative upper bound on the difference between Broker and Gateway /// UTC. The operator must maintain synchronized clocks. #[serde(default = "default_uncertainty")] pub clock_uncertainty_ms: u64, } +/// The worker sandbox the operator chose (ADR 0001). The Gateway runs every +/// grant worker under it, and never under the other one. +#[derive(Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub(crate) enum WorkerSandbox { + /// The base layers inside bubblewrap's namespaces. + #[default] + Namespaces, + /// The base layers alone, for a host that refuses namespaces. + Filter, +} + +impl WorkerSandbox { + /// The layer the supervisor adds around the base layers, when it adds one. + pub fn outer_layer(self) -> Option<&'static str> { + match self { + WorkerSandbox::Namespaces => Some("namespaces"), + WorkerSandbox::Filter => None, + } + } +} + +impl fmt::Display for WorkerSandbox { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + WorkerSandbox::Namespaces => "namespaces", + WorkerSandbox::Filter => "filter", + }) + } +} + fn default_uncertainty() -> u64 { 1000 } diff --git a/crates/countersign-gateway/src/runtime/mod.rs b/crates/countersign-gateway/src/runtime/mod.rs index ceaea2f..4f926e6 100644 --- a/crates/countersign-gateway/src/runtime/mod.rs +++ b/crates/countersign-gateway/src/runtime/mod.rs @@ -118,7 +118,7 @@ impl Gateway { return Err(Capacity.into()); } let (fence, closed) = watch::channel(false); - let (process, exit) = ChildHandle::launch()?; + let (process, exit) = ChildHandle::launch(self.config.worker_sandbox)?; let slot = Slot { fence, requests: Arc::new(Semaphore::new(32)), @@ -856,9 +856,10 @@ pub fn grant_worker() -> Result<()> { match countersign_gateway::worker_sandbox::confine() { Ok(layers) => worker::serve(layers), Err(error) => { - // The startup probe passes this on: the layer, then the cause. - eprintln!("{error}"); - std::process::exit(1); + // The startup probe passes both on: the cause as written here, + // the layer as the exit status. + eprintln!("{}", error.source); + std::process::exit(worker::failed_layer_exit(error.layer)); } } } @@ -875,8 +876,19 @@ pub async fn run(args: &[String]) -> Result<()> { if verb != "serve" { return Err(invalid("unrecognized Gateway command")); } - let layers = ChildHandle::verify_environment().await?; - eprintln!("Gateway worker sandbox \"namespaces\": namespaces, {layers}"); + let sandbox = config.worker_sandbox; + let layers = match ChildHandle::verify_environment(sandbox).await { + Ok(layers) => layers, + Err(error) => { + // In its own words, not as `main` would print it. + eprintln!("{error}"); + std::process::exit(1); + } + }; + match sandbox.outer_layer() { + Some(outer) => eprintln!("Gateway worker sandbox \"{sandbox}\": {outer}, {layers}"), + None => eprintln!("Gateway worker sandbox \"{sandbox}\": {layers}"), + } let policy = config.policy()?; let broker = Broker::new(&config)?; let (issuer, tls) = if let Some(shared) = &config.shared { diff --git a/crates/countersign-gateway/src/runtime/worker.rs b/crates/countersign-gateway/src/runtime/worker.rs index ce571be..11accfc 100644 --- a/crates/countersign-gateway/src/runtime/worker.rs +++ b/crates/countersign-gateway/src/runtime/worker.rs @@ -1,7 +1,7 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use countersign_gateway::{ upstream::{self, SigningRelay}, - worker_sandbox::{Landlock, Layers}, + worker_sandbox::{Landlock, Layer, Layers}, }; use countersign_protocol::{ Origin, @@ -34,7 +34,7 @@ use std::{ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_rustls::TlsAcceptor; -use super::{Result, forward, invalid}; +use super::{Result, config::WorkerSandbox, forward, invalid}; #[derive(Serialize, Deserialize)] pub struct Start { @@ -102,9 +102,126 @@ pub async fn receive(socket: &mut tokio::net::UnixStream) - Ok(serde_json::from_slice(&buf)?) } -/// Only public runtime libraries and the executable enter this grant's new -/// mount, PID and network namespaces. Each inherited stream is private IPC. -fn sandbox(executable: &std::path::Path) -> Command { +/// The command that starts a grant worker under the chosen worker sandbox. +/// Either way it gets a cleared environment, and it confines itself with the +/// base layers once it runs. +fn launch_command(sandbox: WorkerSandbox, executable: &std::path::Path) -> Command { + match sandbox { + WorkerSandbox::Namespaces => namespace_sandbox(executable), + WorkerSandbox::Filter => filter_sandbox(executable), + } +} + +/// The filter sandbox: the supervisor starts the grant worker itself. +fn filter_sandbox(executable: &std::path::Path) -> Command { + use std::os::unix::process::CommandExt; + let mut command = Command::new(executable); + command.env_clear(); + let supervisor = std::process::id() as nix::libc::pid_t; + // SAFETY: the closure runs between fork and exec, and makes only system + // calls: no allocation, no locks. + unsafe { + command.pre_exec(move || { + use nix::libc; + let check = |result: libc::c_long| match result { + -1 => Err(std::io::Error::last_os_error()), + _ => Ok(()), + }; + // What bubblewrap's --die-with-parent does under the namespace + // sandbox: a grant worker does not outlive its supervisor, whose + // end it would not see on its control socket. The supervisor may + // have ended before the request took effect. + check(libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL).into())?; + if libc::getppid() != supervisor { + return Err(std::io::Error::from_raw_os_error(libc::ESRCH)); + } + // Only the control socket and the standard streams stay open + // across the exec, whatever the supervisor itself inherited. + // Marked and not closed here: the standard library reports a + // failed exec over a descriptor of its own. + let marked = check(libc::syscall( + libc::SYS_close_range, + 3 as libc::c_uint, + libc::c_uint::MAX, + libc::CLOSE_RANGE_CLOEXEC, + )); + // Linux before 5.11 has no such flag, and an older container + // profile refuses the call. + marked.or_else(|_| mark_each_close_on_exec()) + }); + } + command +} + +/// Mark every descriptor above the standard streams close-on-exec, one at a +/// time, as listed by the kernel. It fails when it cannot list or mark them. +/// +/// # Safety +/// Makes only system calls, with no allocation and no locks, so it may run +/// between fork and exec, where the process has one thread. +unsafe fn mark_each_close_on_exec() -> std::io::Result<()> { + use nix::libc; + let failed = std::io::Error::last_os_error; + // SAFETY (the whole body): every pointer stays inside `entries`, which + // the kernel filled with `read` bytes of whole directory entries, each + // with a NUL-terminated name. `dir` is open from here to the close below. + unsafe { + let dir = libc::open( + c"/proc/self/fd".as_ptr(), + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC, + ); + if dir < 0 { + return Err(failed()); + } + // Of 64-bit words: a directory entry starts with one. + let mut entries = [0u64; 512]; + loop { + let read = libc::syscall( + libc::SYS_getdents64, + dir, + entries.as_mut_ptr(), + std::mem::size_of_val(&entries), + ); + if read == 0 { + break; + } + if read < 0 { + return Err(failed()); + } + let mut at = 0; + while at < read as usize { + let entry = entries + .as_ptr() + .cast::() + .add(at) + .cast::(); + at += (*entry).d_reclen as usize; + // The name is the descriptor's number. `.` and `..` are not one. + let mut digit = (&raw const (*entry).d_name).cast::(); + let mut fd: libc::c_int = 0; + while (*digit).is_ascii_digit() { + fd = fd * 10 + libc::c_int::from(*digit - b'0'); + digit = digit.add(1); + } + if *digit != 0 || fd <= 2 { + continue; + } + if libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) < 0 { + return Err(failed()); + } + } + } + // Itself close-on-exec, and closed on every other path by the exit + // that follows a failed launch. + libc::close(dir); + } + Ok(()) +} + +/// The namespace sandbox: only public runtime libraries and the executable +/// enter this grant's new mount, PID and network namespaces. Each inherited +/// stream is private IPC. +fn namespace_sandbox(executable: &std::path::Path) -> Command { let mut command = Command::new("/usr/bin/bwrap"); command .env_clear() @@ -128,6 +245,36 @@ fn sandbox(executable: &std::path::Path) -> Command { command } +/// Why the startup probe failed: the chosen worker sandbox, the layer that +/// failed and the cause. It names no other sandbox: the operator reads the +/// trade-off before choosing one (ADR 0001, decision 4). +#[derive(Debug, thiserror::Error)] +#[error( + "worker sandbox \"{sandbox}\" failed: {layer}: {cause}\nSee \"Choose a worker sandbox\" in the operator guide: https://zeroae.github.io/countersign/operate/#choose-a-worker-sandbox" +)] +pub struct SandboxError { + sandbox: WorkerSandbox, + layer: String, + cause: String, +} + +/// The exit status of a grant worker that could not apply a base layer. It +/// carries the layer to the startup probe, through bubblewrap too, which +/// exits with the status of the grant worker it ran. The cause is on the +/// standard error stream. +pub const fn failed_layer_exit(layer: Layer) -> i32 { + match layer { + Layer::NonDumpable => 64, + Layer::Landlock => 65, + Layer::AllowList => 66, + } +} +fn failed_layer(status: ExitStatus) -> Option { + [Layer::NonDumpable, Layer::Landlock, Layer::AllowList] + .into_iter() + .find(|layer| status.code() == Some(failed_layer_exit(*layer))) +} + /// Resolves with a grant worker's exit status when it ends on its own, and /// with an error when the supervisor stopped it. pub type Exit = tokio::sync::oneshot::Receiver; @@ -164,21 +311,25 @@ pub struct ChildHandle { _stop: tokio::sync::oneshot::Sender<()>, } impl ChildHandle { - fn start(stderr: Stdio) -> Result<(tokio::process::Child, UnixDatagram)> { + fn start( + sandbox: WorkerSandbox, + stderr: Stdio, + ) -> Result<(tokio::process::Child, UnixDatagram)> { let (control, child_control) = UnixDatagram::pair()?; control.set_nonblocking(true)?; - let process = tokio::process::Command::from(sandbox(&std::env::current_exe()?)) - .kill_on_drop(true) - .arg("grant-worker") - .stdin(Stdio::from(OwnedFd::from(child_control))) - .stdout(Stdio::null()) - .stderr(stderr) - .spawn()?; + let process = + tokio::process::Command::from(launch_command(sandbox, &std::env::current_exe()?)) + .kill_on_drop(true) + .arg("grant-worker") + .stdin(Stdio::from(OwnedFd::from(child_control))) + .stdout(Stdio::null()) + .stderr(stderr) + .spawn()?; Ok((process, control)) } - pub fn launch() -> Result<(Self, Exit)> { - let (mut process, control) = Self::start(Stdio::null())?; + pub fn launch(sandbox: WorkerSandbox) -> Result<(Self, Exit)> { + let (mut process, control) = Self::start(sandbox, Stdio::null())?; let (stop, mut stopped) = tokio::sync::oneshot::channel::<()>(); let (report, exit) = tokio::sync::oneshot::channel(); tokio::spawn(async move { @@ -202,14 +353,45 @@ impl ChildHandle { )) } - /// Check the actual host sandbox, executable and control socket at startup. - /// This short-lived worker receives no grant, credentials or traffic. It - /// confines itself as every grant worker does and reports its layers. - pub async fn verify_environment() -> Result { - let (process, control) = Self::start(Stdio::piped())?; - control.send(&[0])?; - let output = - tokio::time::timeout(Duration::from_secs(5), process.wait_with_output()).await??; + /// Check the chosen worker sandbox, the executable and the control socket + /// at startup. This short-lived worker receives no grant, credentials or + /// traffic. It confines itself as every grant worker does and reports its + /// layers. + pub async fn verify_environment( + sandbox: WorkerSandbox, + ) -> std::result::Result { + Self::probe(sandbox) + .await + .map_err(|(layer, cause)| SandboxError { + sandbox, + layer: match layer { + Some(base) => base.to_string(), + // Not a base layer's failure, so the supervisor's: the + // layer it adds, or under the filter sandbox, which adds + // none, the launch itself. + None => sandbox.outer_layer().unwrap_or("launch").to_owned(), + }, + cause, + }) + } + + /// The probe's layers, or the base layer that failed, when it was one, + /// and the cause. + async fn probe(sandbox: WorkerSandbox) -> std::result::Result, String)> { + let supervisor = |error: &dyn std::fmt::Display| (None, error.to_string()); + let (process, control) = + Self::start(sandbox, Stdio::piped()).map_err(|error| supervisor(&error))?; + control.send(&[0]).map_err(|error| supervisor(&error))?; + let output = tokio::time::timeout(Duration::from_secs(5), process.wait_with_output()) + .await + .map_err(|_| supervisor(&"the probe grant worker did not finish in 5 seconds"))? + .map_err(|error| supervisor(&error))?; + if killed_by_filter(output.status) { + return Err(( + Some(Layer::AllowList), + "the allow-list killed the probe grant worker (SIGSYS)".to_owned(), + )); + } if !output.status.success() { let hint: String = String::from_utf8_lossy(&output.stderr) .split_whitespace() @@ -218,17 +400,20 @@ impl ChildHandle { .chars() .take(512) .collect(); - return Err(std::io::Error::other(format!( - "Gateway worker sandbox failed to start ({}): {hint}", - output.status - )) - .into()); + return Err(( + failed_layer(output.status), + if hint.is_empty() { + format!("the probe grant worker ended with {}", output.status) + } else { + hint + }, + )); } // The report was queued before the probe exited. let mut message = [0u8; 2]; match control.recv(&mut message) { Ok(1) => Ok(reported([message[0]])), - _ => Err(invalid("grant worker reported no worker sandbox layers")), + _ => Err(supervisor(&"the probe grant worker reported no layers")), } } @@ -503,6 +688,31 @@ mod tests { use super::*; use std::{fs, net::TcpListener, path::Path}; #[test] + fn filter_sandbox_launch_passes_on_no_descriptor_and_no_environment() { + // A descriptor the supervisor inherited open across exec, as `dup` + // leaves one. + let file = fs::File::open("/dev/null").unwrap(); + // SAFETY: `dup` takes a descriptor number and returns a new one, closed below. + let inherited = unsafe { nix::libc::dup(std::os::fd::AsRawFd::as_raw_fd(&file)) }; + assert!(inherited > 2); + // sh and env stand in for the grant worker, started as the supervisor starts one. + let descriptors = filter_sandbox(Path::new("/bin/sh")) + .args(["-c", r#"test ! -e /proc/self/fd/"$1""#, "probe"]) + .arg(inherited.to_string()) + .stdin(Stdio::null()) + .output() + .unwrap(); + // SAFETY: closes the descriptor `dup` returned above, once. + unsafe { nix::libc::close(inherited) }; + assert!(descriptors.status.success(), "{descriptors:?}"); + assert!(std::env::vars_os().next().is_some()); + let environment = filter_sandbox(Path::new("/usr/bin/env")).output().unwrap(); + assert!( + environment.status.success() && environment.stdout.is_empty(), + "{environment:?}" + ); + } + #[test] fn worker_namespace_excludes_parent_files_and_parent_network() { let marker = std::env::temp_dir().join(format!("countersign-isolation-{}", uuid::Uuid::new_v4())); @@ -511,7 +721,7 @@ mod tests { let port = listener.local_addr().unwrap().port().to_string(); // Use bash only as the test executable, under the same namespace // and mount configuration as a production grant worker. - let result=sandbox(Path::new("/bin/bash")) + let result=namespace_sandbox(Path::new("/bin/bash")) .args(["-c", r#"test ! -e "$1" && test ! -e /proc/1/mem && ! (echo probe > /dev/tcp/127.0.0.1/"$2")"#,"probe"]) .arg(&marker).arg(port).output(); fs::remove_file(marker).unwrap(); diff --git a/crates/countersign-gateway/tests/runtime.py b/crates/countersign-gateway/tests/runtime.py index 48b9ee3..06c2cd1 100644 --- a/crates/countersign-gateway/tests/runtime.py +++ b/crates/countersign-gateway/tests/runtime.py @@ -1,7 +1,10 @@ """Exercise real Linux Gateway processes against a test-only grant authority and mTLS site. Usage: python3 crates/countersign-gateway/tests/runtime.py target/debug/countersign-gateway -Requires bubblewrap, libnss-wrapper, curl with HTTPS proxy support, and python-cryptography. + [--worker-sandbox namespaces|filter] [--startup-fails LAYER] +Requires libnss-wrapper, curl with HTTPS proxy support, and python-cryptography, and bubblewrap +for the namespace sandbox. --startup-fails is for a host that refuses LAYER of the chosen worker +sandbox: the Gateway has to exit with the error that names the sandbox and that layer. No host DNS or trust store modifications. """ import base64 @@ -179,7 +182,38 @@ def server(addr, handler, tls): thread.join() -def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=False): +def startup_failure(binary, config, sandbox, layer): + """A host that refuses the chosen worker sandbox stops the Gateway at startup.""" + other = {'namespaces': 'filter', 'filter': 'namespaces'}[sandbox] + result = subprocess.run([binary, 'serve', str(config)], capture_output=True, text=True, timeout=20) + sys.stderr.write(result.stderr) + assert result.returncode != 0, 'The Gateway started under a worker sandbox this host was expected to refuse' + assert not result.stdout, result.stdout + lines = result.stderr.splitlines() + assert len(lines) == 2, lines + # The sandbox, the layer that failed, then the cause. + assert re.fullmatch(rf'worker sandbox "{sandbox}" failed: {re.escape(layer)}: \S.*', lines[0]), lines + assert 'operate/#choose-a-worker-sandbox' in lines[1], lines + # Neither a fallback nor a hint at one: the other value is never named. + assert f'"{other}"' not in result.stderr, lines + + +def configuration_loading(dir, binary, config): + """`worker_sandbox` is optional and takes two values. Anything else fails loading.""" + path = dir / 'worker-sandbox.json' + without = {key: value for key, value in config.items() if key != 'worker_sandbox'} + def loads(**field): + path.write_text(json.dumps({**without, **field})) + return subprocess.run([binary, 'env', str(path)], capture_output=True, timeout=10).returncode == 0 + assert loads() + for value in ('namespaces', 'filter'): + assert loads(worker_sandbox=value), value + for value in ('seccomp', 'Filter', 'none', '', None, True, ['filter']): + assert not loads(worker_sandbox=value), value + + +def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=False, sandbox=None, + startup_fails=None): host = f'site.test:{site.server_port}' broker.host = host broker.accounts = {} @@ -221,7 +255,12 @@ def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=Fa else: (dir/'sidecar.token').write_text(token(bindings['alice'])) config.update(workload=bindings['alice'],workload_token=str(dir/'sidecar.token')) + if sandbox: + config['worker_sandbox'] = sandbox (dir/'config.json').write_text(json.dumps(config)) + if startup_fails: + return startup_failure(binary, dir/'config.json', sandbox or 'namespaces', startup_fails) + configuration_loading(dir, binary, config) hosts = dir/'hosts' hosts.write_text('127.0.0.1 localhost\n'+ip+' site.test\n') # Override /etc/hosts only for the Gateway parent. Grant workers are then @@ -241,6 +280,12 @@ def follow(): def wait_for_log(pattern, timeout=4): with logged: return logged.wait_for(lambda: any(re.fullmatch(pattern, line) for line in log), timeout) + def alive(pid): + # A killed process stays listed until its new parent collects it. + try: + return Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0] != 'Z' + except FileNotFoundError: + return False def descendants(pid): child_list = Path(f'/proc/{pid}/task/{pid}/children') direct = [int(c) for c in child_list.read_text().split()] @@ -257,7 +302,8 @@ def descendants(pid): else: raise RuntimeError('Gateway did not start') # Startup names the worker sandbox and every layer its probe worker applied. - assert wait_for_log(r'Gateway worker sandbox "namespaces": namespaces, non-dumpable, ' + assert wait_for_log(('Gateway worker sandbox "filter": ' if sandbox == 'filter' else + 'Gateway worker sandbox "namespaces": namespaces, ') + 'non-dumpable, ' r'Landlock (ABI v[1-9]\d*|unavailable), seccomp allow-list'), log def request(user,extra=(),path='/', timeout=8): args=['curl','--silent','--show-error','--fail','--max-time',str(timeout),'--noproxy','','--proxy',proxy, @@ -266,11 +312,22 @@ def request(user,extra=(),path='/', timeout=8): args+=['--proxy-header','Proxy-Authorization: Bearer '+token(bindings[user])] return subprocess.run(args + list(extra) + ['https://'+host+path], capture_output=True, timeout=timeout+2) users = list(identities) if shared else ['alice'] + def launched_directly(): + # The filter sandbox starts each grant worker itself: every process + # under the Gateway runs the Gateway's executable, started by the Gateway. + workers = descendants(child.pid) + assert workers, 'grant worker missing' + for pid in workers: + status = Path(f'/proc/{pid}/status').read_text() + assert re.search(rf'^PPid:\t{child.pid}$', status, re.M), status + assert re.search(r'^Name:\tcountersign-gat', status, re.M), status if filter_kill: # Only Alice's grant worker runs. A grant worker the allow-list # kills dies of SIGSYS, so deliver that signal to it: the # supervisor cannot tell the two apart. assert request('alice').returncode == 0 + if sandbox == 'filter': + launched_directly() leaves = [pid for pid in descendants(child.pid) if not descendants(pid)] assert len(leaves) == 1, leaves os.kill(leaves[0], signal.SIGSYS) @@ -281,6 +338,16 @@ def request(user,extra=(),path='/', timeout=8): # A fresh approval is a new grant, which gets a new grant worker. broker.accounts[token(bindings['alice'])][2].update(grant='alice-approved-again') assert request('alice').returncode == 0, 'A fresh approval did not restore service' + # No grant worker outlives its supervisor, however the supervisor ends. + workers = descendants(child.pid) + assert len(workers) >= 2, workers + child.kill() + for _ in range(40): + if not any(alive(pid) for pid in workers): + break + threading.Event().wait(.05) + else: + raise AssertionError('A grant worker outlived its supervisor') return with concurrent.futures.ThreadPoolExecutor(max_workers=len(users)) as pool: results = list(pool.map(request,users)) @@ -288,6 +355,8 @@ def request(user,extra=(),path='/', timeout=8): assert res.returncode==0,(user,res.stderr) expected=identities[user][1].fingerprint(hashes.SHA256()).hex().encode() assert res.stdout==expected,(user,res.stdout) + if sandbox == 'filter': + launched_directly() expected_events = {'mtls_accepted','ingress_closed','egress_closed'} with broker.observation_change: for user in users: @@ -350,6 +419,12 @@ def request(user,extra=(),path='/', timeout=8): def main(): binary=str(Path(sys.argv[1]).resolve()) + options = sys.argv[2:] + assert len(options) % 2 == 0 and set(options[::2]) <= {'--worker-sandbox', '--startup-fails'}, __doc__ + options = dict(zip(options[::2], options[1::2])) + sandbox = options.get('--worker-sandbox') + assert sandbox in (None, 'namespaces', 'filter'), __doc__ + startup_fails = options.get('--startup-fails') with tempfile.TemporaryDirectory(prefix='countersign-gateway-test-') as td: os.umask(0o077) dir=Path(td) @@ -371,9 +446,12 @@ def tls(name): ip=s.getsockname()[0] with server(ip,Site,site_tls) as site, server('127.0.0.1',Broker,tls('broker')) as broker: for shared in (False,True): - do_test(dir,binary,ip,site,broker,identities,ca,shared) - print(('shared' if shared else 'sidecar')+' Gateway mTLS passed') - do_test(dir,binary,ip,site,broker,identities,ca,True,filter_kill=True) + do_test(dir,binary,ip,site,broker,identities,ca,shared,sandbox=sandbox,startup_fails=startup_fails) + print(('shared' if shared else 'sidecar')+(' Gateway refused to start' if startup_fails + else ' Gateway mTLS passed')) + if startup_fails: + return + do_test(dir,binary,ip,site,broker,identities,ca,True,filter_kill=True,sandbox=sandbox) print('filter-killed grant worker fenced') diff --git a/docs/develop.md b/docs/develop.md index 61eabb7..c115d18 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -25,13 +25,15 @@ pnpm lint pnpm test ``` +To run the Gateway's tests under the filter sandbox on a host that refuses namespaces, run `scripts/filter-sandbox-container.sh`, which needs Docker. It runs the containment test and the runtime test in a container under Docker's default security profile, as the end-to-end workflow's `gateway-filter-sandbox` job does. Its two last steps use a profile with one quirk each: where the host refuses the allow-list, the filter sandbox has to fail at startup by name, and where the kernel has no `close_range`, a grant worker still has to start with no descriptor of the supervisor's. + For the graphics and network path, run the [demo walkthrough](try.md) with `--verify` to automate the installed Companion and browser. The hosted end-to-end job also runs a separate harness that selects identities from candidate lists of different sizes, verifies the peer fingerprint and checks revocation. The native qualification jobs generate identities in disposable provider stores. On macOS and Windows use a disposable runner account for that harness because it changes user-level keychain or certificate-store state. [End-to-end workflow](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/.github/workflows/e2e.yml) · [Native qualification workflow](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/.github/workflows/native.yml) ## A grant worker died of SIGSYS The worker sandbox's filter kills a grant worker with `SIGSYS` when it makes a system call outside the allow-list, or a listed call with an argument the list refuses. One call is the exception: `clone3` fails as unsupported and kills nothing. The Gateway logs `grant worker killed by the worker sandbox's filter (SIGSYS); grant fenced`, which does not say which call it was. Two commands do. -The kernel writes an audit line for each filter kill. Reading it needs no root where your user can read the system journal, as members of `adm` or `systemd-journal` can. +The kernel writes an audit line for each filter kill, under either worker sandbox. Reading it needs no root where your user can read the system journal, as members of `adm` or `systemd-journal` can. ```sh journalctl -k | grep type=1326 diff --git a/docs/operate.md b/docs/operate.md index 2fe15d8..b7a74bd 100644 --- a/docs/operate.md +++ b/docs/operate.md @@ -11,7 +11,7 @@ docker build -f distribution/Dockerfile --target gateway -t countersign-gateway: ## Choose the network and trust boundaries -Each parent calls the Integrator's Broker over TLS. The Companion maintains its own outbound authenticated connection to that Broker. The parent retains the real downstream and upstream sockets. A worker receives connected local streams and creates the downstream leaf key. It negotiates both TLS legs and asks the Broker, through its parent, for a scoped client signature. A worker has private mount, PID and network namespaces. A new approval gets its own worker, including when the same user selects the same certificate for another Context. +Each parent calls the Integrator's Broker over TLS. The Companion maintains its own outbound authenticated connection to that Broker. The parent retains the real downstream and upstream sockets. A worker receives connected local streams and creates the downstream leaf key. It negotiates both TLS legs and asks the Broker, through its parent, for a scoped client signature. The [worker sandbox](#choose-a-worker-sandbox) keeps each worker from files, the network, the supervisor and other workers. A new approval gets its own worker, including when the same user selects the same certificate for another Context. ### Sidecar @@ -19,10 +19,10 @@ Each parent calls the Integrator's Broker over TLS. The Companion maintains its 1. Put the Gateway and its one Workload on a private link. Bind `listen` only to the Workload-facing interface. Anyone who can connect to a sidecar listener can act as its Workload. In the Compose demo the upstream site shares the Gateway's backend network and cannot connect to its proxy port. 2. Mount the Workload credential and configuration into the Gateway. Give the Gateway write access to a separate public trust directory. Mount that directory read-only at the same path in the Workload. Use separate principals and PID/mount namespaces, remove tracing privileges and keep the container engine socket outside the Workload. Protect the host's memory and swap according to the sensitivity of the traffic. -3. Start `countersign-gateway serve gateway.json`. Startup probes an actual sandboxed worker. The sidecar creates a new CA and atomically publishes its public PEM at `ca_output`. Its CA key lives in parent memory until exit. The CA encodes critical DNS name constraints and excludes all IPv4 and IPv6 ranges. A wildcard constraint uses the bare DNS suffix, such as `corp.example`. The Gateway separately checks the exact port, label depth and apex policy. Client software is responsible for enforcing the constraint. The Gateway enforces destination policy and issues an exact-host leaf on every connection. +3. Start `countersign-gateway serve gateway.json`. Startup probes an actual grant worker under the chosen worker sandbox and logs the sandbox and its layers. The sidecar creates a new CA and atomically publishes its public PEM at `ca_output`. Its CA key lives in parent memory until exit. The CA encodes critical DNS name constraints and excludes all IPv4 and IPv6 ranges. A wildcard constraint uses the bare DNS suffix, such as `corp.example`. The Gateway separately checks the exact port, label depth and apex policy. Client software is responsible for enforcing the constraint. The Gateway enforces destination policy and issues an exact-host leaf on every connection. 4. Run `countersign-gateway env gateway.json` in the launcher and pass its exports to the Workload. Restrict direct egress if the job must use Countersign for a site. Check that the actual Workload UID can read the PEM and reach the proxy, and that it cannot read the private credential, Gateway memory or descriptors. The demo launcher performs these checks before sending curl traffic. -Keep the Linux supervisor unprivileged. It requires `/usr/bin/bwrap` and permission to create user, mount, PID and network namespaces. On Ubuntu with restricted user namespaces the distribution supplies an opt-in `bwrap-userns-restrict` AppArmor profile in `apparmor-profiles`. Check both the host policy and the container's seccomp/AppArmor policy. The demo's container exceptions apply only to its Gateway. [Compose deployment and access checks](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/examples/demo-integrator/compose_demo.py) · [Service mounts and networks](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/examples/demo-integrator/compose.yaml) +Keep the Linux supervisor unprivileged. With the default worker sandbox it requires `/usr/bin/bwrap` and permission to create user, mount, PID and network namespaces. On Ubuntu with restricted user namespaces the distribution supplies an opt-in `bwrap-userns-restrict` AppArmor profile in `apparmor-profiles`. Check both the host policy and the container's seccomp/AppArmor policy. The demo's container exceptions apply only to its Gateway. [Compose deployment and access checks](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/examples/demo-integrator/compose_demo.py) · [Service mounts and networks](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/examples/demo-integrator/compose.yaml) ### Shared Gateway @@ -43,6 +43,38 @@ Choose a local issuing CA or an ACM-PCA-compatible service. For the local source Maintain affinity between a live grant and the Gateway supervisor that first admitted it. A replacement process has a fresh boot identity. Retire the old grant and obtain a fresh approval when moving its Workload. The two-user integration fixture exercises concurrent workers and cutoff on Linux. A production shared deployment, its confinement policy and a live issuing service need qualification in their target environment. +## Choose a worker sandbox + +The Gateway runs each grant in its own grant worker, a short-lived process that holds that grant's traffic keys. The worker sandbox is the OS isolation around every grant worker. Set it with the Gateway field `worker_sandbox`. Both values work in sidecar and shared mode, and the supervisor stays unprivileged under both: neither needs the Gateway to run as root. + +| `worker_sandbox` | Use it when | What isolates a grant worker | +| --- | --- | --- | +| `"namespaces"` (default, recommended) | The host lets an unprivileged process create user, mount, PID and network namespaces. | The base layers, inside namespaces that bubblewrap creates: the grant worker has its own filesystem, process and network view. Requires `/usr/bin/bwrap`. | +| `"filter"` | The host refuses namespaces and you cannot change that, as on ECS Fargate. | The base layers alone. Does not use bubblewrap. | + +Every grant worker applies the same base layers to itself before it reads any input: + +- a seccomp allow-list of the system calls a grant worker needs, which kills the grant worker on any other call. It cannot open a file, open a socket, run a program or signal another process; +- Landlock, at the version the kernel offers, as a second refusal of file access, and of TCP and signals on newer kernels. A kernel without Landlock still starts, and the startup log says so; +- non-dumpable memory, so no other process of the same user can read it; +- no inherited environment, and no open descriptors beyond its control socket and standard streams. + +Use the namespace sandbox wherever the host allows it. The filter sandbox gives up the namespace layer: a grant worker shares the supervisor's filesystem, process table and network stack, and only the base layers stand between it and them. Under the namespace sandbox a flaw in one base layer still leaves the grant worker in empty namespaces. Under the filter sandbox it does not. + +The Gateway never switches sandboxes on its own. Startup runs a grant worker under the sandbox you chose and logs what it applied: + +```text +Gateway worker sandbox "namespaces": namespaces, non-dumpable, Landlock ABI v7, seccomp allow-list +``` + +When the sandbox cannot start, the Gateway exits with the sandbox, the layer that failed and the cause, and serves nothing: + +```text +worker sandbox "namespaces" failed: namespaces: bwrap: No permissions to create new namespace +``` + +The layer is `namespaces` when bubblewrap could not run or the host refused its namespaces, `launch` when the filter sandbox could not start the grant worker, and otherwise the base layer that failed: `non-dumpable`, `Landlock` or `seccomp allow-list`. First try to fix what the host refused, using the [troubleshooting table](#audit-and-troubleshoot). Change `worker_sandbox` only after reading the trade-off above. [ADR 0001](https://github.com/zeroae/countersign/blob/main/docs/adr/0001-filter-sandbox-for-hosts-without-namespaces.md) records the decision and its evidence. + ## Start the Broker and issue credentials Create a service-owned SQLite state directory and private files for the backend token and Workload HMAC key. For shared mode add a Gateway service token. Provision an independent administrator token to enable stop and recovery operations. Keep each secret at least 32 bytes long and distinct. Put only public client-certificate trust anchors in `client_ca_file`. @@ -99,7 +131,7 @@ sqlite3 -readonly /state/broker.db \ | Symptom | Where to look | | --- | --- | -| Gateway exits with a worker sandbox startup error | Run under the intended service UID. Check bubblewrap availability, user namespace policy, library mounts and the container security profile. Verify the packaged Gateway rather than testing as root. | +| Gateway exits with `worker sandbox "" failed: : ` | Run under the intended service UID, and verify the packaged Gateway rather than testing as root. The layer says what the host refused. Under `"namespaces"`, layer `namespaces`: check that `/usr/bin/bwrap` exists, the host's user namespace policy, the library mounts and the container's seccomp and AppArmor profile. Under `"filter"`, layer `launch`: check that the Gateway can run its own executable. Under either, layer `non-dumpable`, `Landlock` or `seccomp allow-list`: the container security profile refuses a call the grant worker confines itself with, or the kernel lacks seccomp filters. A host that cannot create namespaces at all needs a deliberate choice: read [Choose a worker sandbox](#choose-a-worker-sandbox). | | CONNECT returns 407 | Shared clients need an HTTPS proxy and one proxy bearer header. Check the token audience, expiry, broker boot, and separate service credential. | | CONNECT returns 403 or keeps asking for approval | Compare rule ID, version, port, grant scope, memberships and stop controls. A denied or expired prompt requires an explicit retry. Check that the Companion is paired, online and can access the selected identity. | | CONNECT returns 429, 503 or 504 | Inspect limits per grant/user, Broker availability, worker fencing, clock synchronization and pending approval. Capture the safe `Countersign-Error` and `Countersign-Request-ID` response headers for correlation. | diff --git a/docs/reference/config.md b/docs/reference/config.md index 74e7b06..6dae8f1 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -8,7 +8,7 @@ The Broker and Gateway take a JSON file as a positional argument. Paths in the f | --- | --- | | `countersign-broker /config/broker.json` | Opens the database, starts a new authority epoch, applies configuration and listens. | | `countersign-broker --recover-backup /config/broker.json` | Runs offline after a restore. Fences previous authority and leaves emergency stop enabled. | -| `countersign-gateway serve /config/gateway.json` | Checks Linux worker sandbox startup, opens the configured issuer, publishes the sidecar public CA when applicable, and listens. | +| `countersign-gateway serve /config/gateway.json` | Probes the chosen worker sandbox and exits if it cannot start, opens the configured issuer, publishes the sidecar public CA when applicable, and listens. | | `countersign-gateway env /config/gateway.json` | Writes shell-quoted exports using `proxy_url` and `ca_output`. The launcher supplies these values to the Workload. | For the sidecar configuration on this page, the helper exports these four values. @@ -113,6 +113,7 @@ This sidecar example assumes the Gateway's private Workload-facing address is `1 | `destinations` | Required | Array of Gateway entries. A sidecar needs at least one to construct its CA constraints. | | `denied_networks` | `[]` | Additional IPv4 or IPv6 CIDRs to block across destinations. | | `clock_uncertainty_ms` | `1000` | Allowed UTC difference from the Broker, in milliseconds, from 0 through 29999. The Gateway subtracts the margin from absolute deadlines and checks its local clock against monotonic elapsed time. | +| `worker_sandbox` | `"namespaces"` | The worker sandbox for every grant worker, in sidecar and shared mode. `"namespaces"` is the namespace sandbox: bubblewrap gives each grant worker its own filesystem, process and network view, and the grant worker then confines itself with a seccomp allow-list, Landlock and non-dumpable memory. It needs `/usr/bin/bwrap` and a host that lets the Gateway create namespaces. `"filter"` is the filter sandbox: the same confinement without the namespaces, for a host that refuses them, such as ECS Fargate. It needs no bubblewrap. Any other value fails loading. The Gateway never switches from one to the other. [Choose a worker sandbox](../operate.md#choose-a-worker-sandbox) | | `shared` | Omitted → sidecar | Shared-mode object. Requires PEM `listener_cert` and `listener_key`, a `gateway_token` file holding the Broker's matching service secret, and a `ca` source below. Omit both sidecar Workload fields. | ### Destination entries diff --git a/docs/specs/2026-11-17-countersign-design.md b/docs/specs/2026-11-17-countersign-design.md index ec0dc00..d6062b9 100644 --- a/docs/specs/2026-11-17-countersign-design.md +++ b/docs/specs/2026-11-17-countersign-design.md @@ -231,7 +231,7 @@ Deliver the slices together on one implementation branch and one pull request in ## Run the reference implementation -The desktop can run on Linux, macOS or Windows. The Gateway requires Linux with bubblewrap and unprivileged user namespaces. The Broker uses a single owning process per SQLite database. Use Node 24, the repository’s pinned Rust toolchain, and pnpm 10.28.2. On Debian-family systems install the NSS development libraries before building the addon. +The desktop can run on Linux, macOS or Windows. The Gateway requires Linux and one of two worker sandboxes, chosen by the operator: the namespace sandbox, which needs bubblewrap and unprivileged user namespaces, or the filter sandbox, for hosts that refuse namespaces (ADR 0001). The Broker uses a single owning process per SQLite database. Use Node 24, the repository’s pinned Rust toolchain, and pnpm 10.28.2. On Debian-family systems install the NSS development libraries before building the addon. ```sh pnpm install --frozen-lockfile diff --git a/scripts/filter-sandbox-container.sh b/scripts/filter-sandbox-container.sh new file mode 100755 index 0000000..83ab08a --- /dev/null +++ b/scripts/filter-sandbox-container.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Prove the filter sandbox on a host that refuses namespaces, as ECS Fargate +# does: run the Gateway's containment test and its runtime test in a container +# under Docker's default security profile, as an unprivileged user. Two last +# steps each use a profile with one quirk instead: a host that refuses the +# allow-list, and a kernel without close_range. +# +# Usage: scripts/filter-sandbox-container.sh +# Needs Docker and the Rust toolchain. Builds on this host and runs what it +# built in the container, so the image matches the CI runner's distribution. +set -euo pipefail +cd "$(dirname "$0")/.." + +# The executable cargo builds for one target of the Gateway crate: the program +# itself, or with `test` the target's tests. A test's build lists the program +# beside it, which the test runs. +built() { + local tests=False + [ "$1" = test ] && tests=True + cargo "$@" --locked -p countersign-gateway --message-format=json | python3 -c ' +import json, sys +found = [message["executable"] for message in map(json.loads, sys.stdin) + if message.get("executable") and message["profile"]["test"] is '"$tests"'] +assert len(found) == 1, found +print(found[0])' +} +gateway=$(built build --bin countersign-gateway) +containment=$(built test --no-run --test worker_sandbox) +units=$(built test --no-run --bin countersign-gateway) +target=$(cargo metadata --locked --no-deps --format-version 1 | + python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])') + +# Two images. The first has no bubblewrap, so the filter sandbox is shown not +# to need it. The second adds it, so the namespace sandbox fails there because +# the host refuses namespaces, not because a program is missing. +build() { + docker build --quiet --target "$1" --tag "countersign-filter-sandbox:$1" - <<'EOF' +FROM ubuntu:24.04 AS filter +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl libnss-wrapper python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* \ + && pip install --no-cache-dir --break-system-packages cryptography==50.0.2 +FROM filter AS namespaces +RUN apt-get update && apt-get install -y --no-install-recommends bubblewrap && rm -rf /var/lib/apt/lists/* +EOF +} +build filter +build namespaces + +# With `profile` empty: no --privileged, --cap-add or --security-opt, which is +# Docker's default profile. The checkout and cargo's target directory are +# mounted where they are on this host, because the containment test holds the +# Gateway executable's path from its build. +profile=() +inside() { + local image=$1 + shift + docker run --rm --user "$(id -u):$(id -g)" --workdir "$PWD" \ + --volume "$PWD:$PWD:ro" --volume "$target:$target:ro" \ + "${profile[@]}" "countersign-filter-sandbox:$image" "$@" +} +# A profile that allows every system call but the one these rules refuse, for +# the steps that need a host with one quirk. +rules=$(mktemp) +trap 'rm -f "$rules"' EXIT +refusing() { + printf '{"defaultAction": "SCMP_ACT_ALLOW", "syscalls": [%s]}\n' "$1" >"$rules" + profile=(--security-opt "seccomp=$rules") +} +runtime=(timeout 120 python3 crates/countersign-gateway/tests/runtime.py "$gateway") + +echo '== The container refuses namespaces' +if inside namespaces bwrap --unshare-all --ro-bind / / -- /usr/bin/true; then + echo 'This container can create namespaces, so it proves nothing about the filter sandbox.' >&2 + exit 1 +fi +if inside filter test -e /usr/bin/bwrap; then + echo 'The image for the filter sandbox has bubblewrap.' >&2 + exit 1 +fi + +echo '== Worker sandbox containment test' +inside filter timeout 120 "$containment" + +echo '== Runtime test under the filter sandbox, without bubblewrap' +inside filter "${runtime[@]}" --worker-sandbox filter + +echo '== The namespace sandbox fails at startup with the named error' +inside namespaces "${runtime[@]}" --worker-sandbox namespaces --startup-fails namespaces + +# The other direction, in the image that has bubblewrap: where the host +# refuses the allow-list, the filter sandbox has to fail by name, and not +# start grant workers some other way. 22 is both PR_SET_SECCOMP and EINVAL. +echo '== The filter sandbox fails at startup with the named error' +refusing '{"names": ["prctl"], "action": "SCMP_ACT_ERRNO", "errnoRet": 22, + "args": [{"index": 0, "value": 22, "op": "SCMP_CMP_EQ"}]}' +inside namespaces "${runtime[@]}" --worker-sandbox filter --startup-fails 'seccomp allow-list' + +# Linux before 5.11 cannot mark every descriptor close-on-exec in one call. +# 38 is ENOSYS, as such a kernel answers. +echo '== The filter sandbox passes on no descriptor on a kernel without close_range' +refusing '{"names": ["close_range"], "action": "SCMP_ACT_ERRNO", "errnoRet": 38}' +inside filter timeout 60 "$units" filter_sandbox_launch