diff --git a/crates/countersign-gateway/src/runtime/config.rs b/crates/countersign-gateway/src/runtime/config.rs index a73a6d9..4a486b2 100644 --- a/crates/countersign-gateway/src/runtime/config.rs +++ b/crates/countersign-gateway/src/runtime/config.rs @@ -25,6 +25,12 @@ pub(crate) struct Config { pub workload_token: Option, pub workload: Option, pub destinations: Vec, + /// The HTTP proxy every destination goes through, unless its rule says + /// `"egress": "direct"` (ADR 0003). + #[serde(rename = "egress_proxy")] + egress_proxy_url: Option, + #[serde(skip)] + pub egress_proxy: Option, #[serde(default)] pub denied_networks: Vec, pub shared: Option, @@ -67,6 +73,77 @@ impl fmt::Display for WorkerSandbox { } } +/// The egress proxy's address, as the operator wrote it: a name or an IP +/// literal, resolved on every connection. +#[derive(Debug, Clone)] +pub(crate) struct EgressProxy { + pub host: String, + pub port: u16, +} + +impl EgressProxy { + fn parse(url: &str) -> Result { + let shape = || { + invalid( + "egress_proxy must be an http URL with a host and a port, such as http://proxy.internal:3128", + ) + }; + let parsed = reqwest::Url::parse(url).map_err(|_| shape())?; + if parsed.scheme() == "https" { + return Err(invalid( + "egress_proxy must be an http URL: the Gateway sends CONNECT to the egress proxy in plain text and does not support an https egress proxy", + )); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err(invalid( + "egress_proxy must not carry credentials: the Gateway does not authenticate to the egress proxy", + )); + } + // The URL parser forgets a port it knows as the default, so look for + // one in the text: after the host, outside an IPv6 literal's brackets. + let authority = url + .split_once("://") + .map_or("", |(_, rest)| rest) + .split(['/', '?', '#']) + .next() + .unwrap_or_default(); + let written_port = authority + .rsplit(']') + .next() + .and_then(|tail| tail.rsplit_once(':')) + .is_some_and(|(_, port)| !port.is_empty()); + let (Some(host), Some(port)) = (parsed.host_str(), parsed.port_or_known_default()) else { + return Err(shape()); + }; + if parsed.scheme() != "http" + || !written_port + || port == 0 + || parsed.path() != "/" + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(shape()); + } + Ok(Self { + host: host + .trim_start_matches('[') + .trim_end_matches(']') + .to_owned(), + port, + }) + } +} + +impl fmt::Display for EgressProxy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.host.contains(':') { + write!(f, "[{}]:{}", self.host, self.port) + } else { + write!(f, "{}:{}", self.host, self.port) + } + } +} + fn default_uncertainty() -> u64 { 1000 } @@ -90,14 +167,43 @@ pub(crate) struct Rule { pub approval: ApprovalMode, #[serde(default)] pub private_networks: Vec, + #[serde(default)] + pub egress: Egress, pub roots: PathBuf, #[serde(default)] pub revocation: Option, } +/// A destination's route to the site (ADR 0003). +#[derive(Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub(crate) enum Egress { + /// Through the egress proxy when the Gateway has one, else direct. The + /// operator never writes it: it is what a rule without `egress` means. + #[default] + #[serde(skip_deserializing)] + Default, + /// Always direct, with the Gateway's own address policy. + Direct, +} + impl Config { pub fn load(path: &Path) -> Result { - let config: Self = serde_json::from_slice(&fs::read(path)?)?; + let mut config: Self = serde_json::from_slice(&fs::read(path)?)?; + config.egress_proxy = config + .egress_proxy_url + .as_deref() + .map(EgressProxy::parse) + .transpose()?; + if let Some(rule) = config.destinations.iter().find(|rule| { + config.egress_proxy_for(rule).is_some() && !rule.private_networks.is_empty() + }) { + return Err(io::Error::other(format!( + "destination rule {:?} has private_networks and goes through the egress proxy, which does not check them: remove private_networks, or set \"egress\": \"direct\" on the rule", + rule.id + )) + .into()); + } let sidecar = config.shared.is_none(); if config.workload.is_some() != sidecar || config.workload_token.is_some() != sidecar @@ -130,6 +236,33 @@ impl Config { Ok(config) } + /// The egress proxy this rule's destinations go through, if any. + pub fn egress_proxy_for(&self, rule: &Rule) -> Option<&EgressProxy> { + match rule.egress { + Egress::Direct => None, + Egress::Default => self.egress_proxy.as_ref(), + } + } + + /// Which destinations leave their address policy to the egress proxy, in + /// words for the startup log. + pub fn left_to_egress_proxy(&self) -> String { + let ids = self + .destinations + .iter() + .filter(|rule| self.egress_proxy_for(rule).is_some()) + .map(|rule| rule.id.as_str()) + .collect::>(); + if ids.is_empty() { + "carries the address policy of no destination".to_owned() + } else { + format!( + "carries the address policy of destinations: {}", + ids.join(", ") + ) + } + } + pub fn policy(&self) -> Result { if self .destinations diff --git a/crates/countersign-gateway/src/runtime/egress.rs b/crates/countersign-gateway/src/runtime/egress.rs new file mode 100644 index 0000000..e53bc24 --- /dev/null +++ b/crates/countersign-gateway/src/runtime/egress.rs @@ -0,0 +1,120 @@ +//! The proxied leg to a site: a plain-text `CONNECT` through the egress proxy +//! (ADR 0003). The egress proxy resolves the site's name and carries its +//! address policy; the Gateway resolves only the proxy. + +use std::{error::Error, fmt, net::SocketAddr}; + +use countersign_protocol::Origin; +use ipnet::IpNet; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, +}; + +use super::{Result, config::EgressProxy, invalid}; + +/// The most the Gateway reads of the egress proxy's answer before the tunnel. +const MAX_HEAD: usize = 8 * 1024; + +/// The egress proxy answered the `CONNECT` with something other than 2xx. +#[derive(Debug)] +pub struct ProxyAnswer(pub u16); + +impl fmt::Display for ProxyAnswer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "the egress proxy answered {}", self.0) + } +} + +impl Error for ProxyAnswer {} + +/// The `Countersign-Error` value for a proxied leg that failed (ADR 0003). +pub fn error_value(error: &(dyn Error + Send + Sync + 'static)) -> &'static str { + match error.downcast_ref::() { + Some(ProxyAnswer(400..=499)) => "proxy_denied", + // As a direct route reports a site it cannot reach. + Some(ProxyAnswer(500..=599)) => "upstream_failure", + // Unreachable, resolved only inside `denied_networks`, out of time + // (the deadline covers the whole leg), or no usable HTTP answer. + _ => "proxy_unavailable", + } +} + +/// Opens a tunnel to `origin` through the egress proxy. On success the stream +/// carries the site's bytes from its first byte: the Gateway reads the +/// proxy's answer no further than the end of its head. +pub async fn tunnel(proxy: &EgressProxy, origin: &Origin, denied: &[IpNet]) -> Result { + // Resolved on every connection, so a load balancer may change addresses. + // Only `denied_networks` applies: the proxy normally sits on a private one. + let addresses = tokio::net::lookup_host((proxy.host.as_str(), proxy.port)) + .await? + .filter(|address| { + let ip = address.ip().to_canonical(); + !denied.iter().any(|block| block.contains(&ip)) + }) + .collect::>(); + if addresses.is_empty() { + return Err(invalid("egress proxy resolved to no allowed addresses")); + } + let mut stream = TcpStream::connect(&addresses[..]).await?; + stream + .write_all(format!("CONNECT {origin} HTTP/1.1\r\nHost: {origin}\r\n\r\n").as_bytes()) + .await?; + match answer(&mut stream).await? { + 200..=299 => Ok(stream), + code => Err(ProxyAnswer(code).into()), + } +} + +/// Reads the egress proxy's answer to the `CONNECT` and returns its status +/// code. It reads past the status line only for a 2xx answer, up to the head's +/// end, and gives up as soon as the bytes cannot be an HTTP answer: a proxy +/// that keeps the connection open after any other answer gets no wait. +async fn answer(stream: &mut TcpStream) -> Result { + const VERSION: &[u8] = b"HTTP/1."; + let not_http = || invalid("egress proxy gave no HTTP answer"); + let mut line = Vec::with_capacity(64); + let mut code = None; + for _ in 0..MAX_HEAD { + // One byte at a time, so no byte of the tunnel is consumed. + let byte = stream.read_u8().await?; + if byte != b'\n' { + line.push(byte); + if code.is_none() && !line.starts_with(VERSION) && !VERSION.starts_with(&line) { + return Err(not_http()); + } + continue; + } + // A line ends at LF, with or without CR before it (RFC 9112, 2.2). + if line.last() == Some(&b'\r') { + line.pop(); + } + match code { + None => { + let status = status(&line).ok_or_else(not_http)?; + if !(200..300).contains(&status) { + return Ok(status); + } + code = Some(status); + } + Some(status) if line.is_empty() => return Ok(status), + Some(_) => {} + } + line.clear(); + } + Err(invalid("egress proxy answer exceeds the limit")) +} + +/// The status code of a status line, given without its line end. +fn status(line: &[u8]) -> Option { + let line = std::str::from_utf8(line).ok()?; + let mut parts = line.splitn(3, ' '); + if !matches!(parts.next()?, "HTTP/1.1" | "HTTP/1.0") { + return None; + } + let code = parts.next()?; + if code.len() != 3 || !code.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + code.parse().ok() +} diff --git a/crates/countersign-gateway/src/runtime/mod.rs b/crates/countersign-gateway/src/runtime/mod.rs index 4f926e6..37875d5 100644 --- a/crates/countersign-gateway/src/runtime/mod.rs +++ b/crates/countersign-gateway/src/runtime/mod.rs @@ -1,5 +1,6 @@ mod broker; mod config; +mod egress; mod forward; mod relay; mod worker; @@ -386,26 +387,32 @@ fn admission_status(error: &(dyn Error + Send + Sync + 'static)) -> StatusCode { StatusCode::SERVICE_UNAVAILABLE } fn http_response(status: StatusCode) -> ReplyBody { - let mut builder = Response::builder().status(status); - if !status.is_success() { - let code = match status { - StatusCode::PROXY_AUTHENTICATION_REQUIRED => "authentication_required", - StatusCode::FORBIDDEN => "denied", - StatusCode::TOO_MANY_REQUESTS => "busy", - StatusCode::GATEWAY_TIMEOUT => "timeout", - StatusCode::SERVICE_UNAVAILABLE => "unavailable", - StatusCode::BAD_GATEWAY => "upstream_failure", - _ => "invalid_request", - }; - let request_id = id(); - builder = builder - .header("Countersign-Error", code) - .header("Countersign-Request-ID", request_id.as_str()); - if status == StatusCode::PROXY_AUTHENTICATION_REQUIRED { - builder = builder.header(header::PROXY_AUTHENTICATE, "Bearer"); - } - eprintln!("Gateway CONNECT {} {code}", request_id.as_str()); + if status.is_success() { + let builder = Response::builder().status(status); + return builder.body(Empty::new()).expect("fixed response"); + } + let code = match status { + StatusCode::PROXY_AUTHENTICATION_REQUIRED => "authentication_required", + StatusCode::FORBIDDEN => "denied", + StatusCode::TOO_MANY_REQUESTS => "busy", + StatusCode::GATEWAY_TIMEOUT => "timeout", + StatusCode::SERVICE_UNAVAILABLE => "unavailable", + StatusCode::BAD_GATEWAY => "upstream_failure", + _ => "invalid_request", + }; + error_response(status, code) +} +/// A refused CONNECT: no body, the error value, and the request ID the log line shares. +fn error_response(status: StatusCode, code: &str) -> ReplyBody { + let request_id = id(); + let mut builder = Response::builder() + .status(status) + .header("Countersign-Error", code) + .header("Countersign-Request-ID", request_id.as_str()); + if status == StatusCode::PROXY_AUTHENTICATION_REQUIRED { + builder = builder.header(header::PROXY_AUTHENTICATE, "Bearer"); } + eprintln!("Gateway CONNECT {} {code}", request_id.as_str()); builder.body(Empty::new()).expect("fixed response") } async fn connect_request(gateway: Arc, req: Request) -> ReplyBody { @@ -484,13 +491,27 @@ async fn connect_request(gateway: Arc, req: Request) -> Reply let Ok(trust) = PeerTrust::load(rule) else { return http_response(StatusCode::BAD_GATEWAY); }; + let egress_proxy = gateway.config.egress_proxy_for(rule); let target = timeout_at(deadline, async { - let pinned = config::resolve_target(&origin, rule, &gateway.config.denied_networks).await?; - Ok::<_, Box>(TcpStream::connect(pinned).await?) + let denied = &gateway.config.denied_networks; + // Never both: a proxied destination is never tried directly. + match egress_proxy { + Some(proxy) => egress::tunnel(proxy, &origin, denied).await, + None => { + let pinned = config::resolve_target(&origin, rule, denied).await?; + Ok(TcpStream::connect(pinned).await?) + } + } }) - .await; - let Ok(Ok(target)) = target else { - return http_response(StatusCode::BAD_GATEWAY); + .await + .map_err(Into::into) + .and_then(|target| target); + let target = match target { + Ok(target) => target, + Err(error) if egress_proxy.is_some() => { + return error_response(StatusCode::BAD_GATEWAY, egress::error_value(error.as_ref())); + } + Err(_) => return http_response(StatusCode::BAD_GATEWAY), }; if ingress.guard.check().is_err() || egress.guard.check().is_err() { return http_response(StatusCode::SERVICE_UNAVAILABLE); @@ -868,7 +889,14 @@ pub async fn run(args: &[String]) -> Result<()> { let [verb, path] = args else { return Err(invalid("usage: countersign-gateway serve|env CONFIG.json")); }; - let config = Config::load(Path::new(path))?; + let config = match Config::load(Path::new(path)) { + Ok(config) => config, + Err(error) => { + // In its own words, so a cause that quotes a field reads as written. + eprintln!("{error}"); + std::process::exit(1); + } + }; if verb == "env" { println!("{}", config.env()?); return Ok(()); @@ -890,6 +918,13 @@ pub async fn run(args: &[String]) -> Result<()> { None => eprintln!("Gateway worker sandbox \"{sandbox}\": {layers}"), } let policy = config.policy()?; + // Only once the destinations it names have passed their checks. + if let Some(proxy) = &config.egress_proxy { + eprintln!( + "Gateway egress proxy {proxy} {}", + config.left_to_egress_proxy() + ); + } let broker = Broker::new(&config)?; let (issuer, tls) = if let Some(shared) = &config.shared { let issuer = SharedIssuer::open(shared.ca.clone(), policy.clone()).await?; diff --git a/crates/countersign-gateway/tests/runtime.py b/crates/countersign-gateway/tests/runtime.py index 19a5789..6979f0c 100644 --- a/crates/countersign-gateway/tests/runtime.py +++ b/crates/countersign-gateway/tests/runtime.py @@ -1,4 +1,5 @@ -"""Exercise real Linux Gateway processes against a test-only grant authority and mTLS site. +"""Exercise real Linux Gateway processes against a test-only grant authority and mTLS site, +directly and through a test-only egress proxy. Usage: python3 crates/countersign-gateway/tests/runtime.py target/debug/countersign-gateway [--worker-sandbox namespaces|filter] [--startup-fails LAYER] @@ -19,6 +20,7 @@ from pathlib import Path import signal import socket +import socketserver import ssl import subprocess import sys @@ -80,9 +82,49 @@ def token(workload): return 'fixture.' + base64.urlsafe_b64encode(body).decode().rstrip('=') + '.fixture' -class Fixture(http.server.ThreadingHTTPServer): +class Counting: + """A server that records the peer of each connection it accepts, before it reads a byte of it.""" + + def counted(self, peer): + with self.accepted: + self.peers.append(peer) + self.accepted.notify_all() + + @property + def connections(self): + return len(self.peers) + + def accepted_since(self, seen): + """The connections the server accepted after its first `seen`, other than a probe of ours. It + accepts in order, so once it has accepted the probe it has accepted every connection before it.""" + with socket.create_connection(self.server_address[:2], timeout=4) as probe: + mine = probe.getsockname() + with self.accepted: + if not self.accepted.wait_for(lambda: mine in self.peers[seen:], 4): + # Only an earlier connection holds up the server's accept. + assert self.peers[seen:], 'The probe was not accepted' + return [peer for peer in self.peers[seen:] if peer != mine] + + +class Fixture(Counting, http.server.ThreadingHTTPServer): daemon_threads = True + def __init__(self, address, handler, tls): + super().__init__(address, handler) + self.tls = tls + self.peers = [] + self.accepted = threading.Condition() + + def get_request(self): + # Counted before the TLS handshake, so a connection the handshake ends counts too. + connection, address = self.socket.accept() + self.counted(address) + try: + return self.tls.wrap_socket(connection, server_side=True), address + except OSError: + connection.close() + raise + def handle_error(self, request, client_address): # Cancelling an outstanding poll or forcibly cutting off either TLS leg # can reset the fixture socket while the HTTP library is reading it. @@ -129,7 +171,7 @@ def do_POST(self): if token not in authority.accounts or not self.headers.get('X-Countersign-Gateway-Instance'): self.send_error(403) return - user, key, grant = authority.accounts[token] + user, key, grants = authority.accounts[token] if authority.shared and self.headers.get('X-Countersign-Gateway') != authority.service_secret: self.send_error(403) return @@ -137,26 +179,31 @@ def do_POST(self): body = frame['body'] now = int(time.time() * 1000) msg = frame['type'] - if msg in ('connect', 'lookup') and body['workload'] == grant['workload'] and body['host'] == authority.host: + if msg == 'sign': + # Every signature it is asked for, granted or not. + authority.signatures.append(body['binding']['host']) + # One grant for each destination the user can reach. + grant = grants.get(body.get('host') or body.get('binding', {}).get('host')) + if msg in ('connect', 'lookup') and grant and body['workload'] == grant['workload']: kind = 'activate' value = {'activation': {'grant': grant, 'offer': user, 'approval': user, 'expires_at_ms': authority.expires_at}, 'chain': [authority.public[user]]} - elif msg == 'watch' and body == grant: + elif msg == 'watch' and body in grants.values(): retired = authority.retired[user].wait(5) - kind, value = ('retire' if retired else 'no_change'), grant - elif msg == 'observed' and body['grant'] == grant and body['host'] == authority.host: + kind, value = ('retire' if retired else 'no_change'), body + elif msg == 'observed' and grant and body['grant'] == grant: kind = 'observed' value = body with authority.observation_change: received = authority.observations.setdefault((user,body['tls_connection']),set()) received.add(body['observation']) authority.observation_change.notify_all() - elif msg == 'renew_lease' and body['grant'] == grant: + elif msg == 'renew_lease' and body['grant'] in grants.values(): kind = 'lease' budget = 2000 if user == 'alice' else 30000 value = {'binding': body, 'disposition': {'granted': {'budget_ms': budget, 'valid_until_ms': now + budget}}} - elif msg == 'sign' and body['binding']['grant'] == grant and body['binding']['host'] == authority.host: + elif msg == 'sign' and grant and body['binding']['grant'] == grant: assert body['tls'] == 'tls13' and body['scheme'] == 'ecdsa_nistp256_sha256' transcript = base64.b64decode(body['certificate_verify'], validate=True) assert transcript.startswith(b' ' * 64 + b'TLS 1.3, client CertificateVerify\x00') @@ -177,8 +224,7 @@ def log_message(self, *args): @contextlib.contextmanager def server(addr, handler, tls): - httpd = Fixture((addr, 0), handler) - httpd.socket = tls.wrap_socket(httpd.socket, server_side=True) + httpd = Fixture((addr, 0), handler, tls) thread = threading.Thread(target=httpd.serve_forever, daemon=True) thread.start() try: @@ -189,6 +235,111 @@ def server(addr, handler, tls): thread.join() +# Like Squid's, each refusal carries a page of the proxy's own. A Workload never sees it. +PAGE = b'test egress proxy error page\n' +ERROR_PAGE = b'content-type: text/html\r\ncontent-length: %d\r\n\r\n%s' % (len(PAGE), PAGE) + + +# Names whose proxied leg fails, one for each way: the test-only egress proxy refuses, closes +# unanswered, answers with something that is not HTTP, cannot reach the site, or tunnels to a +# site whose certificate is for another name. For `banner`, `stub` and `lf` it answers and then +# keeps the connection open: a line that is not HTTP, a few bytes that are not, and a refusal +# whose lines end in a bare LF. +PROXY_FAILURES = ['refused', 'closed', 'garbled', 'banner', 'stub', 'lf', 'unreachable', 'wrong'] + + +class KeptOpen(bytes): + """An answer after which the test-only egress proxy keeps the connection open until the + Gateway closes it.""" + + +class Tunnel(socketserver.BaseRequestHandler): + """One CONNECT through the test-only egress proxy. Like Squid, it ends both legs when either ends.""" + + def handle(self): + head = b'' + while b'\r\n\r\n' not in head: + data = self.request.recv(4096) + if not data or len(head) > 8192: + return + head += data + line = head.split(b'\r\n', 1)[0].decode('latin-1') + with self.server.recorded: + self.server.requests.append(line) + method, target, _ = (line.split(' ', 2) + ['', ''])[:3] + host, _, port = target.rpartition(':') + address = self.server.names.get(host) + if method != 'CONNECT' or address is None: + self.request.sendall(b'HTTP/1.1 403 Forbidden\r\n' + ERROR_PAGE) + return + if isinstance(address, bytes): + # This answer and no other, then the end of the connection: the proxy's, or for an + # answer kept open, the Gateway's. + self.request.sendall(address) + if isinstance(address, KeptOpen): + with contextlib.suppress(OSError): + self.request.recv(1) + return + try: + site = socket.create_connection(address if isinstance(address, tuple) else (address, int(port)), + timeout=5) + except OSError: + self.request.sendall(b'HTTP/1.1 503 Service Unavailable\r\n' + ERROR_PAGE) + return + with site: + site.settimeout(None) + self.request.sendall(b'HTTP/1.1 200 Connection established\r\n\r\n') + def pump(source, sink): + with contextlib.suppress(OSError): + while data := source.recv(65536): + sink.sendall(data) + for leg in (source, sink): + with contextlib.suppress(OSError): + leg.shutdown(socket.SHUT_RDWR) + back = threading.Thread(target=pump, args=(site, self.request), daemon=True) + back.start() + pump(self.request, site) + back.join() + + +class EgressProxy(Counting, socketserver.ThreadingTCPServer): + """A test-only egress proxy. It records each request line, and is the only resolver of `names`. + A name it maps to bytes gets those bytes as its whole answer, and one it maps to an address and + port is tunneled there, whatever port the CONNECT names.""" + daemon_threads = True + + def __init__(self, names): + super().__init__(('127.0.0.1', 0), Tunnel) + self.names = names + self.requests = [] + self.recorded = threading.Lock() + self.peers = [] + self.accepted = threading.Condition() + + def get_request(self): + request, peer = super().get_request() + self.counted(peer) + return request, peer + + def take(self): + with self.recorded: + requests, self.requests = self.requests, [] + return requests + + +@contextlib.contextmanager +def egress_proxy(names): + proxy = EgressProxy(names) + thread = threading.Thread(target=proxy.serve_forever, daemon=True) + thread.start() + try: + yield proxy + finally: + proxy.shutdown() + proxy.server_close() + thread.join() + + def startup_failure(binary, config, sandbox, layer): """A host that refuses the chosen worker sandbox stops the Gateway at startup.""" other = {'namespaces': 'filter', 'filter': 'namespaces'}[sandbox] @@ -219,14 +370,77 @@ def loads(**field): assert not loads(worker_sandbox=value), value +def egress_configuration(dir, binary, config): + """`egress_proxy` is optional: an http URL with a host and a port. `env` and `serve` refuse + the same configurations, and say why.""" + path = dir / 'egress.json' + base = {key: value for key, value in config['destinations'][0].items() if key != 'egress'} + rule = {key: value for key, value in base.items() if key != 'private_networks'} + def write(proxy, **fields): + changed = {**config, 'destinations': [{**rule, **fields}]} + if proxy is not None: + changed['egress_proxy'] = proxy + path.write_text(json.dumps(changed)) + def loads(proxy=None, **fields): + write(proxy, **fields) + return subprocess.run([binary, 'env', str(path)], capture_output=True, timeout=10).returncode == 0 + def refused(proxy=None, **fields): + # Both commands, and the same words from each: one line that names the cause. + write(proxy, **fields) + results = [subprocess.run([binary, verb, str(path)], capture_output=True, text=True, timeout=10) + for verb in ('env', 'serve')] + for result in results: + assert result.returncode != 0 and not result.stdout, (proxy, fields, result) + assert results[0].stderr == results[1].stderr, results + lines = results[0].stderr.splitlines() + assert len(lines) == 1, lines + return lines[0] + for proxy in ('http://proxy.test:3128', 'http://127.0.0.1:3128', 'http://[::1]:3128', 'http://proxy.test:3128/'): + assert loads(proxy), proxy + error = refused('https://proxy.test:3128') + assert 'egress_proxy' in error and 'https' in error, error + for proxy in ('http://user:secret@proxy.test:3128', 'http://user@proxy.test:3128'): + error = refused(proxy) + assert 'egress_proxy' in error and 'credentials' in error and 'secret' not in error, error + for proxy in ('http://proxy.test', 'http://proxy.test:', 'http://proxy.test:0', 'http://[::1]', 'http://proxy.test:3128/path', 'http://proxy.test:3128?q', 'ftp://proxy.test:3128', + 'proxy.test:3128', ''): + assert 'egress_proxy' in refused(proxy), proxy + refused(3128) + # The startup line says which destinations leave their address policy to the egress proxy, + # so a configuration whose destinations fail their checks never prints it. + wildcard = {**base, 'id': 'wild', 'pattern': '*.' + base['pattern'].split('.', 1)[1], 'egress': 'direct'} + path.write_text(json.dumps({**config, 'egress_proxy': 'http://proxy.test:3128', 'destinations': [rule, wildcard]})) + result = subprocess.run([binary, 'serve', str(path)], capture_output=True, text=True, timeout=10) + assert result.returncode != 0 and 'exact host and prefix' in result.stderr, result + assert 'Gateway egress proxy' not in result.stderr, result.stderr + # A rule's `egress` takes one value, with or without an egress proxy. + direct = {**base, 'egress': 'direct'} + assert loads(**direct) and loads('http://proxy.test:3128', **direct) + # Nobody checks `private_networks` on a proxied route, so a rule may not claim it. + error = refused('http://proxy.test:3128', **base) + assert re.search(r'\bsite\b', error) and 'private_networks' in error, error + assert 'remove' in error and '"egress": "direct"' in error, error + for value in ('proxy', 'default', 'Direct', 'none', '', None, True, ['direct']): + refused('http://proxy.test:3128', egress=value) + refused(egress=value) + + def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=False, sandbox=None, - startup_fails=None): + startup_fails=None, egress=None, ignored=None, unusable=None): + """With `egress`, a test-only egress proxy, the Gateway reaches the site only through it. With + `ignored` instead, the Gateway's environment names that proxy, and the Gateway connects directly. + With `unusable` as well as `egress`, configuration that leaves the Gateway no usable egress proxy, + the Gateway can resolve and reach the site itself, and must not.""" host = f'site.test:{site.server_port}' - broker.host = host + # The site again, under a name with a direct route beside the egress proxy. + direct = f'direct.test:{site.server_port}' + # Destinations the Gateway and the grant authority allow, whose proxied leg fails. + failing = {name: f'{name}.test:{site.server_port}' for name in PROXY_FAILURES} broker.accounts = {} broker.retired = {user: threading.Event() for user in identities} broker.public = {} broker.observations = {} + broker.signatures = [] broker.observation_change = threading.Condition() broker.shared = shared broker.service_secret = 'secret-' * 8 @@ -237,12 +451,17 @@ def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=Fa principal = 'alice' if user == 'alice-other-context' else user workload = {'integrator':'local','user':principal,'context':user,'workload':user,'workload_generation':1} der = cert.public_bytes(serialization.Encoding.DER) - grant = {'workload':workload,'grant':user,'grant_version':1,'worker':user,'worker_generation':1, - 'scope':{'entry_id':'site','entry_version':1,'pattern':host}, - 'device':{'pairing':user,'key':user,'channel_generation':1}, 'cert_sha256':b64(hashlib.sha256(der).digest())} + def grant(name, entry, pattern): + return {'workload':workload,'grant':name,'grant_version':1,'worker':name,'worker_generation':1, + 'scope':{'entry_id':entry,'entry_version':1,'pattern':pattern}, + 'device':{'pairing':user,'key':user,'channel_generation':1}, + 'cert_sha256':b64(hashlib.sha256(der).digest())} bindings[user] = workload broker.public[user] = b64(der) - broker.accounts[token(workload)] = (user,key,grant) + broker.accounts[token(workload)] = (user,key,{host: grant(user, 'site', host), + direct: grant(user+'-direct', 'direct', direct), + **{to: grant(f'{user}-{name}', name, to) + for name, to in failing.items()}}) with socket.socket() as tmp: tmp.bind(('127.0.0.1',0)) port = tmp.getsockname()[1] @@ -252,10 +471,21 @@ def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=Fa 'broker_url':f'https://localhost:{broker.server_port}', 'broker_ca':str(dir/'ca.pem'), 'clock_uncertainty_ms':100, 'destinations':[{'id':'site','version':1,'pattern':host,'roots':str(dir/'ca.pem'),'private_networks':[ip+'/32']}]} + if egress: + # The site's rule leaves its address policy to the egress proxy. The + # other keeps its own, on a direct route. + config['egress_proxy'] = f'http://localhost:{egress.server_address[1]}' + del config['destinations'][0]['private_networks'] + proxied = config['destinations'][0] + config['destinations'].append({**proxied, 'id':'direct', 'pattern':direct, + 'private_networks':[ip+'/32'], 'egress':'direct'}) + config['destinations'] += [{**proxied, 'id':name, 'pattern':to} for name, to in failing.items()] + config.update(unusable or {}) if shared: crl = dir / 'site.crl' publish_crl(crl, ca, 1, datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5)) - config['destinations'][0]['revocation'] = {'kind':'crl', 'files':[str(crl)]} + for rule in config['destinations']: + rule['revocation'] = {'kind':'crl', 'files':[str(crl)]} config['shared']={'listener_cert':str(dir/'listener.pem'),'listener_key':str(dir/'listener.key'), 'gateway_token':str(dir/'service'), 'ca':{'type':'local','cert':str(dir/'ca.pem'),'key':str(dir/'ca.key')}} @@ -264,17 +494,38 @@ def do_test(dir, binary, ip,site, broker, identities, ca, shared, filter_kill=Fa config.update(workload=bindings['alice'],workload_token=str(dir/'sidecar.token')) if sandbox: config['worker_sandbox'] = sandbox + if shared and not egress and not filter_kill: + # Without an egress proxy, `"egress": "direct"` changes nothing: the + # sidecar pass runs without it, this one with it. + config['destinations'][0]['egress'] = 'direct' (dir/'config.json').write_text(json.dumps(config)) if startup_fails: return startup_failure(binary, dir/'config.json', sandbox or 'namespaces', startup_fails) - configuration_loading(dir, binary, config) + if not egress: + configuration_loading(dir, binary, config) + egress_configuration(dir, binary, config) hosts = dir/'hosts' - hosts.write_text('127.0.0.1 localhost\n'+ip+' site.test\n') + # Behind the egress proxy only the proxy can resolve the site's name. + hosts.write_text('127.0.0.1 localhost\n' + (ip+' direct.test\n' if egress else '') + + ('' if egress and not unusable else ip+' site.test\n')) # Override /etc/hosts only for the Gateway parent. Grant workers are then # isolated by the production launcher and receive only preconnected streams. library = os.environ.get('NSS_WRAPPER_LIBRARY') or str(next(Path('/usr/lib').glob('*/libnss_wrapper.so'))) + environment = {**os.environ,'LD_PRELOAD':library,'NSS_WRAPPER_HOSTS':str(hosts)} + if ignored: + # Only `egress_proxy` chooses the route, never the environment. + environment.update({name: f'http://localhost:{ignored.server_address[1]}' for name in + ('HTTPS_PROXY','https_proxy','HTTP_PROXY','http_proxy','ALL_PROXY','all_proxy')}) + ignored.take() + if egress: + resolved = subprocess.run(['getent','ahosts','site.test'], env=environment, capture_output=True, timeout=10) + if unusable: + assert resolved.returncode == 0 and ip in resolved.stdout.decode(), ('The Gateway could not reach the site', resolved) + else: + assert resolved.returncode == 2, ('The Gateway could resolve the site itself', resolved) + egress.take() with subprocess.Popen([binary,'serve',str(dir/'config.json')], stderr=subprocess.PIPE, text=True, - env={**os.environ,'LD_PRELOAD':library,'NSS_WRAPPER_HOSTS':str(hosts)}) as child: + env=environment) as child: log = [] logged = threading.Condition() def follow(): @@ -312,13 +563,56 @@ def descendants(pid): assert wait_for_log(('Gateway worker sandbox "filter": ' if sandbox == 'filter' else 'Gateway worker sandbox "namespaces": namespaces, ') + 'non-dumpable, ' r'Landlock (ABI v[1-9]\d*|unavailable), seccomp allow-list'), log - def request(user,extra=(),path='/', timeout=8): + if egress: + # Startup names the egress proxy, and the rules whose address policy it now carries. + assert wait_for_log(rf'Gateway egress proxy localhost:{config["egress_proxy"].rpartition(":")[2]} carries the ' + r'address policy of destinations: ' + ', '.join(['site', *failing])), log + def request(user,extra=(),path='/', timeout=8, to=host): args=['curl','--silent','--show-error','--fail','--max-time',str(timeout),'--noproxy','','--proxy',proxy, '--proxy-cacert',str(dir/'ca.pem'),'--cacert',str(pub)] if shared: args+=['--proxy-header','Proxy-Authorization: Bearer '+token(bindings[user])] - return subprocess.run(args + list(extra) + ['https://'+host+path], capture_output=True, timeout=timeout+2) + return subprocess.run(args + list(extra) + ['https://'+to+path], capture_output=True, timeout=timeout+2) + def connect(user, to): + """The Gateway's whole answer to one Workload CONNECT, as bytes.""" + listener = socket.create_connection(('127.0.0.1', port), timeout=8) + if shared: + # As curl does, without Python's stricter X.509 profile. + trust = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + trust.load_verify_locations(dir/'ca.pem') + listener = trust.wrap_socket(listener, server_hostname='localhost') + lines = [f'CONNECT {to} HTTP/1.1', f'Host: {to}', 'Connection: close'] + if shared: + lines.append('Proxy-Authorization: Bearer ' + token(bindings[user])) + with listener: + listener.sendall(('\r\n'.join(lines) + '\r\n\r\n').encode()) + answer = b'' + while data := listener.recv(65536): + answer += data + return answer + def fails_through_egress(to, error): + """The proxied leg to `to` fails: the Workload reads why, and the site sees nothing.""" + seen = site.connections + answer = connect(users[0], to) + # Never a direct connection instead. + assert not site.accepted_since(seen), (to, 'The site saw a connection', answer) + head, _, body = answer.partition(b'\r\n\r\n') + status, *fields = head.decode('latin-1').split('\r\n') + headers = {key.lower(): value for key, _, value in (f.partition(': ') for f in fields)} + assert status.startswith('HTTP/1.1 502 '), (to, answer) + assert headers.get('countersign-error') == error, (to, answer) + request_id = headers.get('countersign-request-id') + assert request_id, (to, answer) + # Nothing of the egress proxy's own answer reaches the Workload. + assert body == b'' and b'egress proxy' not in answer, (to, answer) + assert wait_for_log(rf'Gateway CONNECT {re.escape(request_id)} {error}'), (to, log) users = list(identities) if shared else ['alice'] + if unusable: + seen = egress.connections + fails_through_egress(host, 'proxy_unavailable') + assert not egress.accepted_since(seen), 'The egress proxy saw a connection' + assert egress.take() == [] + return def launched_directly(): # The filter sandbox starts each grant worker itself: every process # under the Gateway runs the Gateway's executable, started by the Gateway. @@ -344,7 +638,7 @@ def launched_directly(): assert request('alice').returncode != 0, 'A grant whose worker the filter killed was served again' assert request('bob').returncode == 0, "Alice's killed grant worker affected Bob" # A fresh approval is a new grant, which gets a new grant worker. - broker.accounts[token(bindings['alice'])][2].update(grant='alice-approved-again') + broker.accounts[token(bindings['alice'])][2][host].update(grant='alice-approved-again') assert request('alice').returncode == 0, 'A fresh approval did not restore service' # No grant worker outlives its supervisor, however the supervisor ends. workers = descendants(child.pid) @@ -363,6 +657,43 @@ def launched_directly(): assert res.returncode==0,(user,res.stderr) expected=identities[user][1].fingerprint(hashes.SHA256()).hex().encode() assert res.stdout==expected,(user,res.stdout) + through_egress = [f'CONNECT {host} HTTP/1.1'] * len(users) + if egress: + # Each Workload CONNECT opened its own tunnel, named by the site's hostname and port. + assert egress.take() == through_egress, through_egress + # The same Gateway reaches the rule with `"egress": "direct"` on its own. + for user in users: + res = request(user, to=direct) + assert res.returncode == 0, (user, res.stderr) + assert res.stdout == identities[user][1].fingerprint(hashes.SHA256()).hex().encode() + assert egress.take() == [], 'A direct destination went through the egress proxy' + # The egress proxy refuses a name it does not allow. + fails_through_egress(failing['refused'], 'proxy_denied') + assert egress.take() == [f'CONNECT {failing["refused"]} HTTP/1.1'], 'The refusal was not the proxy\'s' + # It closes the connection unanswered, or answers with something that is not HTTP. + fails_through_egress(failing['closed'], 'proxy_unavailable') + fails_through_egress(failing['garbled'], 'proxy_unavailable') + assert egress.take() == [f'CONNECT {failing[name]} HTTP/1.1' for name in ('closed', 'garbled')] + # It answers and keeps the connection open. The Workload reads why at once, inside + # `connect`'s timeout, never at the Gateway's 120-second deadline. + fails_through_egress(failing['banner'], 'proxy_unavailable') + fails_through_egress(failing['stub'], 'proxy_unavailable') + fails_through_egress(failing['lf'], 'proxy_denied') + assert egress.take() == [f'CONNECT {failing[name]} HTTP/1.1' for name in ('banner', 'stub', 'lf')] + # It cannot reach the site, and says so as Squid does, with a 5xx answer. + fails_through_egress(failing['unreachable'], 'upstream_failure') + assert egress.take() == [f'CONNECT {failing["unreachable"]} HTTP/1.1'] + # Through a tunnel it allows, to a site whose certificate is for another name, the + # request fails before the user's certificate signs anything. + res = request(users[0], to=failing['wrong']) + assert res.returncode != 0 and not res.stdout, (res.returncode, res.stdout) + assert egress.take() == [f'CONNECT {failing["wrong"]} HTTP/1.1'], 'The egress proxy refused the tunnel' + assert host in broker.signatures, 'The grant authority recorded no signature at all' + assert failing['wrong'] not in broker.signatures, 'A site with the wrong certificate got a signature' + if ignored: + assert ignored.take() == [], 'The Gateway took the egress proxy from its environment' + with logged: + assert not any(line.startswith('Gateway egress proxy') for line in log), log if sandbox == 'filter': launched_directly() expected_events = {'mtls_accepted','ingress_closed','egress_closed'} @@ -393,6 +724,10 @@ def launched_directly(): assert site.cutoff.wait(1), 'upstream leg survived cutoff' assert time.monotonic() - begin < 4, 'cutoff exceeded lease and test scheduling budget' assert request('alice').returncode != 0, 'Lapsed worker reused old grant' + if egress: + # The slow request went through a tunnel too, as did each since. + taken = egress.take() + assert taken and set(taken) == set(through_egress), taken if shared: assert request('bob').returncode == 0, 'Alice cutoff affected Bob' assert request('alice-other-context').returncode == 0, 'Cutoff crossed Contexts using the same identity' @@ -406,6 +741,9 @@ def launched_directly(): broker.retired['bob'].set() assert future.result(timeout=4).returncode != 0 assert site.cutoff.wait(1) + if egress: + # The certificate revalidation below does not depend on the route. + return # Revalidation at 10s imports a new, shorter evidence deadline. # The idle transport must close at that deadline ahead of the # next periodic check and its still-current 30-second leases. @@ -437,7 +775,7 @@ def main(): os.umask(0o077) dir=Path(td) ca=identity(dir,'ca') - for name,hosts in [('broker',['localhost']),('listener',['localhost']),('upstream',['site.test'])]: + for name,hosts in [('broker',['localhost']),('listener',['localhost']),('upstream',['site.test','direct.test'])]: identity(dir,name,ca,hosts) identities={u: identity(dir,u,ca) for u in ('alice','bob')} identities['alice-other-context'] = identities['alice'] @@ -452,13 +790,32 @@ def tls(name): with socket.socket(socket.AF_INET,socket.SOCK_DGRAM) as s: s.connect(('192.0.2.1',9)) ip=s.getsockname()[0] - with server(ip,Site,site_tls) as site, server('127.0.0.1',Broker,tls('broker')) as broker: + with socket.socket() as s: + s.bind(('127.0.0.1',0)) + # Nothing listens here once the socket closes. + nothing = s.getsockname() + with (server(ip,Site,site_tls) as site, server('127.0.0.1',Broker,tls('broker')) as broker, + egress_proxy({'site.test': ip, 'wrong.test': ip, 'closed.test': b'', 'unreachable.test': nothing, + 'garbled.test': b'SSH-2.0-test egress proxy\r\n\r\n', + 'banner.test': KeptOpen(b'SSH-2.0-test egress proxy\r\n'), 'stub.test': KeptOpen(b'SSH-2.0'), + 'lf.test': KeptOpen(b'HTTP/1.1 403 Forbidden\n\n')}) as egress): for shared in (False,True): - do_test(dir,binary,ip,site,broker,identities,ca,shared,sandbox=sandbox,startup_fails=startup_fails) + do_test(dir,binary,ip,site,broker,identities,ca,shared,sandbox=sandbox,startup_fails=startup_fails, + ignored=egress) print(('shared' if shared else 'sidecar')+(' Gateway refused to start' if startup_fails else ' Gateway mTLS passed')) if startup_fails: return + for shared in (False,True): + do_test(dir,binary,ip,site,broker,identities,ca,shared,sandbox=sandbox,egress=egress) + print(('shared' if shared else 'sidecar')+' Gateway mTLS through the egress proxy passed') + # Nothing listens at the egress proxy's address. + do_test(dir,binary,ip,site,broker,identities,ca,shared,sandbox=sandbox,egress=egress, + unusable={'egress_proxy': f'http://localhost:{nothing[1]}'}) + # The egress proxy's only address is inside `denied_networks`. + do_test(dir,binary,ip,site,broker,identities,ca,shared,sandbox=sandbox,egress=egress, + unusable={'denied_networks': ['127.0.0.0/8', '::1/128']}) + print(('shared' if shared else 'sidecar')+' Gateway without a usable egress proxy failed closed') do_test(dir,binary,ip,site,broker,identities,ca,True,filter_kill=True,sandbox=sandbox) print('filter-killed grant worker fenced') diff --git a/docs/adr/0003-the-egress-proxy-resolves-the-site-and-carries-its-address-policy.md b/docs/adr/0003-the-egress-proxy-resolves-the-site-and-carries-its-address-policy.md index 50d15c5..705dd71 100644 --- a/docs/adr/0003-the-egress-proxy-resolves-the-site-and-carries-its-address-policy.md +++ b/docs/adr/0003-the-egress-proxy-resolves-the-site-and-carries-its-address-policy.md @@ -1,5 +1,5 @@ --- -status: proposed +status: accepted date: 2026-11-21 supersedes: none --- diff --git a/docs/architecture.md b/docs/architecture.md index 556beb6..879cdce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ Solid arrows show the application's traffic. Dashed arrows show control connecti ## Follow a request across the two connections 1. **Workload → Gateway.** curl, Python or another HTTPS client asks its configured proxy to CONNECT to a DNS name and port. The Gateway checks the protected destination and Workload binding with the Broker. For an admitted grant, its worker presents a fresh leaf for the exact intercepted host. The Workload validates this leaf against the proxy CA the Integrator supplied. -2. **Gateway → protected site.** The Gateway opens a separate connection to the pinned destination. It authenticates the upstream server under that destination's trust policy. When the server requests client authentication, rustls asks for a signature over the TLS handshake input. +2. **Gateway → protected site.** The Gateway opens a separate connection: directly to the address it resolved and checked, or, when the operator names an egress proxy, through a `CONNECT` tunnel to the site's name that the egress proxy resolves. It authenticates the upstream server under that destination's trust policy. When the server requests client authentication, rustls asks for a signature over the TLS handshake input. 3. **Gateway → Broker → Companion.** The Broker checks the live grant and signing operation, then routes the request to the desktop that approved it. The Companion verifies its local approval and invokes the native key provider. The selected public certificate chain and the resulting signature let the Gateway finish the handshake. The client private key stays in its provider. 4. **Site → Workload.** The site authenticates the client's certificate and applies its own application permissions. The worker relays the response over the downstream TLS connection. The Broker handles authorization and signing messages. Application bodies travel through the Gateway. @@ -19,7 +19,8 @@ Solid arrows show the application's traffic. Dashed arrows show control connecti |---|---|---| | Integrator backend | Authenticated account, Context ownership, job generation, service credential | Bind each job to the correct user and Context, authorize its destinations, deliver credentials privately. | | Workload | Job data, proxy address, public trust bundle. In shared mode it or its private adapter also has a Workload credential. | Use the configured proxy. The Integrator controls direct network access from the job. | -| Gateway | Downstream and upstream traffic keys, plaintext, scoped signing requests. The parent holds issuer access. | Enforce exact authorities, isolate grants, authenticate the upstream site and close both sockets when authority lapses. A compromised Gateway could send other requests to an approved site. | +| Gateway | Downstream and upstream traffic keys, plaintext, scoped signing requests. The parent holds issuer access. | Enforce exact authorities and, on a direct route, the site's address policy. Isolate grants, authenticate the upstream site and close both sockets when authority lapses. A compromised Gateway could send other requests to an approved site. | +| Egress proxy, when the operator names one | The site's hostname and port, and the encrypted bytes of the upstream connection | Resolve the site's name and open a tunnel only to the names and addresses the operator allows: on that route its ACLs are the site's address policy. It cannot read the traffic, and a tunnel to the wrong machine gets no signature, because the Gateway verifies the site's certificate first. [Reach sites through an egress proxy](operate.md#reach-sites-through-an-egress-proxy) | | Broker | Pairings, account bindings, decisions, selected public chain, grants, audit and current signing operations | Authenticate both services and devices, enforce ownership and deadlines, and send a signing request only to its winning device. | | Companion and native provider | Device pairing key and access to installed client identities | Display the actual origin and scope, require consent, and sign with the approved local key. Trust also extends to the user's OS, token provider and desktop session. | | Protected site | Client certificate and the HTTP request it receives | Decide what that certificate can do. The site may issue its own cookies or persist writes beyond the TLS session. | diff --git a/docs/develop.md b/docs/develop.md index 324f8a8..7fa32d6 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -47,6 +47,29 @@ That container has the CI runner's kernel. To run the same two tests on real ECS For the graphics and network path, run the [demo walkthrough](try.md) with `--verify` to automate the installed Companion and browser. The hosted end-to-end job also runs a separate harness that selects identities from candidate lists of different sizes, verifies the peer fingerprint and checks revocation. The native qualification jobs generate identities in disposable provider stores. On macOS and Windows use a disposable runner account for that harness because it changes user-level keychain or certificate-store state. [End-to-end workflow](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/.github/workflows/e2e.yml) · [Native qualification workflow](https://github.com/zeroae/countersign/blob/fbb5aa7c951f8b70a238f3351140aa44a6184adf/.github/workflows/native.yml) +### Qualify the Gateway against a real Squid + +The demo launcher's Squid variant puts a real Squid between the Gateway and the site, as the Gateway's [egress proxy](operate.md#reach-sites-through-an-egress-proxy). Run it by hand before a release that touches the Gateway's route to a site; no workflow runs it. It needs what the [demo walkthrough](try.md) needs, with the same virtual environment, and pulls the Squid image. From the checkout root, on a desktop session: + +```sh +.venv/bin/python examples/demo-integrator/compose_demo.py --squid --verify \ + --app "$PWD/packages/companion/dist/linux-arm64-unpacked/countersign-companion" +``` + +Without a desktop session, run it under a virtual display, as the end-to-end workflow runs its harness: `dbus-run-session -- xvfb-run -a .venv/bin/python examples/demo-integrator/compose_demo.py --squid --verify --app ...`. Without `--verify` the variant runs the interactive walkthrough through Squid. + +`--squid` layers `compose.squid.yaml` over the default `compose.yaml`, which it leaves unchanged. The Gateway shares the backend network with the Broker and Squid, and the site moves to a network only Squid shares. The site runs in the network of a placeholder container, `site-host`, so stopping the site leaves its name and address in place, as for a site whose server is down. The Gateway's configuration names `http://squid:3128` as its `egress_proxy`, and its rule for the site has no `private_networks`. Squid allows `CONNECT` only to `site.test:9444` on the site's address, refuses that name when any of its addresses is another, blocks metadata, link-local and loopback addresses, and does no TLS interception: its configuration is `examples/demo-integrator/squid.conf`. + +`--verify` drives the real Companion and browser, as for the default walkthrough, and fails on the first check that does not hold. Before the approval it checks that the Gateway logged the egress proxy and that it can neither resolve `site.test` nor connect to the site's address. From the approved request it correlates Squid's access log line `TCP_TUNNEL/200` for `CONNECT site.test:9444` to the site's address, the fingerprint the site returned, which must equal the one approved in the Companion, and the response the Workload read. Then, with that grant still live and no second approval, the Workload repeats its request three times. Each must end in 502 with the `Countersign-Error` below, and the Gateway must log it under the same `Countersign-Request-ID`. + +| Step | `Countersign-Error` | What Squid logs | +| --- | --- | --- | +| The launcher swaps Squid's rule to refuse the site and reconfigures it | `proxy_denied` | `TCP_DENIED/403` with `HIER_NONE`: it opened no connection to the site | +| It restores the rule and stops the site | `upstream_failure` | `TCP_TUNNEL/503`: the site's address refused the connection | +| It stops Squid | `proxy_unavailable` | Nothing | + +The variant was qualified against Squid 6.14, as `squid -v` reports it, from the image `ubuntu/squid:6.6-24.04_edge@sha256:d9c34c5ab74c6dd41cf1070d9b4221aac072e67e983527ca2d41512958f77506`, on Linux arm64. The tag names Ubuntu's original Squid version; the image's package is newer. `compose.squid.yaml` pins that digest. To qualify another Squid, change the image there, run the variant and record the version the run prints on its `Egress proxy:` line here. + ## A grant worker died of SIGSYS The worker sandbox's filter kills a grant worker with `SIGSYS` when it makes a system call outside the allow-list, or a listed call with an argument the list refuses. One call is the exception: `clone3` fails as unsupported and kills nothing. The Gateway logs `grant worker killed by the worker sandbox's filter (SIGSYS); grant fenced`, which does not say which call it was. Two commands do. diff --git a/docs/operate.md b/docs/operate.md index b7a74bd..fa1efe2 100644 --- a/docs/operate.md +++ b/docs/operate.md @@ -87,7 +87,73 @@ The authenticated Integrator backend derives the user, Context and Workload inst Configure matching rule IDs, versions, patterns and approval scopes at the Broker and Gateway. An exact rule wins over a wildcard. For example, `*.corp.example:443` matches `api.corp.example:443` only one label below the suffix. The apex, deeper names and other ports require their own entries. By default the prompt covers only the exact host. To offer approval across the wildcard, set `whole_wildcard: true` at the Broker and `approval: "whole_wildcard"` at the Gateway. Each intercepted host still receives its own leaf. -For an internal site, specify its exact DNS name and port and put its allowed addresses in `private_networks`. The parent validates the entire DNS answer set, including IPv4-mapped IPv6, before pinning one socket address. A mixed allowed/forbidden answer fails the connection. Add site-specific infrastructure and control-plane ranges to `denied_networks`. The Gateway also rejects special-use addresses including loopback, link-local, shared carrier space and known metadata endpoints. Within TLS it checks SNI and the authority of each HTTP request. [Destination fields](reference/config.md#destination-entries) +For an internal site on a direct route, specify its exact DNS name and port and put its allowed addresses in `private_networks`. The parent validates the entire DNS answer set, including IPv4-mapped IPv6, before pinning one socket address. A mixed allowed/forbidden answer fails the connection. Add site-specific infrastructure and control-plane ranges to `denied_networks`. The Gateway also rejects special-use addresses including loopback, link-local, shared carrier space and known metadata endpoints. Within TLS it checks SNI and the authority of each HTTP request. [Destination fields](reference/config.md#destination-entries) + +## Reach sites through an egress proxy + +Some networks give the Gateway no direct route to a protected site. The only way out is an HTTP proxy that authorizes each `CONNECT` by hostname and port and passes the TLS bytes through unread. Name that proxy, the egress proxy, once in the Gateway configuration. Every destination then goes through it. A rule that should connect directly says `"egress": "direct"`. Sidecar and shared mode take the same fields. + +```json +{ + "egress_proxy": "http://proxy.internal:3128", + "destinations": [ + {"id": "partner-api", "version": 1, "pattern": "api.partner.example:443", "roots": "/config/partner-roots.pem"}, + {"id": "ledger", "version": 1, "pattern": "ledger.corp.example:443", "roots": "/config/corp-roots.pem", + "egress": "direct", "private_networks": ["10.20.0.0/24"]} + ] +} +``` + +Each route puts name resolution and the site's address policy in a different place. + +| | Direct route | Through the egress proxy | +| --- | --- | --- | +| Rules | Every rule when `egress_proxy` is absent, and a rule with `"egress": "direct"` | Every other rule when `egress_proxy` is set | +| Who resolves the site's name | The Gateway, through system DNS | The egress proxy. The Gateway resolves only the proxy's own name, on every connection, so a load balancer may change addresses | +| Who checks the site's addresses | The Gateway: every DNS answer against `private_networks`, `denied_networks` and its special-use blocks, before it pins one address | The egress proxy's ACLs. The Gateway checks none of the site's addresses | +| `private_networks` | Allowed, for an exact private site | Refused at startup, naming the rule. Remove the field, or set `"egress": "direct"` | + +Everything else stays with Countersign on both routes. A site still needs its destination rule at the Gateway and the Broker, the user still approves, and the grant and its leases cut off a connection in the same way. The Gateway verifies the site's certificate against the hostname and the rule's roots before the user's certificate signs anything, so a tunnel to the wrong machine receives no signature. The egress proxy carries only encrypted bytes. + +On a proxied route the egress proxy's ACLs are the site's address policy, and the Gateway no longer blocks anything for those sites. Configure the egress proxy to: + +- allow `CONNECT` only to the hostnames and ports of your destination rules; +- refuse cloud metadata addresses, such as `169.254.169.254`, `fd00:ec2::254` and `168.63.129.16`, and every link-local and loopback address; +- allow a private site only when every address its name resolves to is inside its own prefixes, where you want the check `private_networks` gave on a direct route; +- pass TLS through without interception. An egress proxy that terminates TLS cannot present the site's certificate, so the request fails. + +In Squid these are `http_access` rules, read from the top, and the first that matches decides. The example below allows `api.partner.example:443`, and a private site, `billing.corp.example:443`, only when all its addresses are in its prefix `10.30.0.0/24`. The rule for each of them has no `private_networks`. The Compose demo's Squid variant qualified the Gateway against these rules, with its one site, pinned to its address, in place of these two ([`squid.conf`](https://github.com/zeroae/countersign/blob/main/examples/demo-integrator/squid.conf)). Docker's DNS gives the demo's site one address, so the refusal of a name with one address inside the prefix and one outside was checked against the same Squid on its own. + +```squid +http_port 3128 + +acl CONNECT method CONNECT +acl tls_port port 443 +acl partner_api dstdomain api.partner.example +acl billing dstdomain billing.corp.example +acl billing_prefixes dst 10.30.0.0/24 +acl billing_outside dst 0.0.0.0-10.29.255.255 10.30.1.0-255.255.255.255 ::/0 +acl to_metadata dst 169.254.169.254 fd00:ec2::254 168.63.129.16 +acl to_link_local dst 169.254.0.0/16 fe80::/10 +acl to_loopback dst 127.0.0.0/8 ::1 + +http_access deny !CONNECT +http_access deny to_metadata +http_access deny to_link_local +http_access deny to_loopback +http_access deny billing billing_outside +http_access allow CONNECT partner_api tls_port +http_access allow CONNECT billing tls_port billing_prefixes +http_access deny all +``` + +The blocks come before the allow rules, so they hold even for an allowed name whose DNS answer is a metadata, link-local or loopback address. A `dst` ACL matches when any one of the addresses a name resolves to is in its list, and Squid may then connect to another of them. `billing_prefixes` alone would therefore let through a name with one address inside the prefix and one outside, and Squid then dialled the outside one. `billing_outside` lists every other address, so the deny on it refuses such a name, as `private_networks` does on a direct route. `http_port` has no `ssl-bump` option, so Squid passes the TLS bytes through unread. Squid 6.14 refuses a `CONNECT` these rules deny with 403, which the Workload reads as `proxy_denied`, and answers 503 when it cannot connect to the site, which the Workload reads as `upstream_failure`. A `dst` ACL does not match a name Squid cannot resolve, so a pinned site whose name does not resolve is refused with 403 too. When the site's address stops answering, Squid answers 503 only after its `connect_timeout`, one minute by default. The Gateway allows a Workload's `CONNECT` 120 seconds in all, the wait for approval included, so a lower `connect_timeout` leaves the Workload time to read `upstream_failure`. + +A wildcard rule behind the egress proxy reaches every host under it that the proxy allows, private ones included: on a direct route the Gateway refuses a wildcard rule for a private site, but through the proxy it cannot tell that a site is private. The Gateway also cannot stop the egress proxy from opening a tunnel to any address its ACLs allow. + +The egress proxy itself normally sits on a private address, such as an internal load balancer. `private_networks` and the special-use blocks do not apply to it. `denied_networks` does: an answer for the proxy's name inside it is never used. + +At startup the Gateway logs the egress proxy and the rules that leave their address policy to it, such as `Gateway egress proxy proxy.internal:3128 carries the address policy of destinations: partner-api`. The Gateway reads the egress proxy only from its configuration, never from `HTTPS_PROXY` or another proxy environment variable, and it always reaches the Broker directly. When the proxied leg fails, the Workload's `CONNECT` returns 502 and the Gateway does not try the site directly. `Countersign-Error` says why: `proxy_denied` when the egress proxy refused the `CONNECT`, `upstream_failure` when it answered that it could not reach the site, and `proxy_unavailable` when the Gateway could not use it at all. The egress proxy's own error page never reaches the Workload. A user can approve and the request then fail at the egress proxy, as it can at an unreachable site on a direct route. [Gateway fields](reference/config.md#gateway) · [Audit and troubleshoot](#audit-and-troubleshoot) · [ADR 0003](https://github.com/zeroae/countersign/blob/main/docs/adr/0003-the-egress-proxy-resolves-the-site-and-carries-its-address-policy.md) ## Maintain trust and stop access @@ -134,6 +200,7 @@ sqlite3 -readonly /state/broker.db \ | Gateway exits with `worker sandbox "" failed: : ` | Run under the intended service UID, and verify the packaged Gateway rather than testing as root. The layer says what the host refused. Under `"namespaces"`, layer `namespaces`: check that `/usr/bin/bwrap` exists, the host's user namespace policy, the library mounts and the container's seccomp and AppArmor profile. Under `"filter"`, layer `launch`: check that the Gateway can run its own executable. Under either, layer `non-dumpable`, `Landlock` or `seccomp allow-list`: the container security profile refuses a call the grant worker confines itself with, or the kernel lacks seccomp filters. A host that cannot create namespaces at all needs a deliberate choice: read [Choose a worker sandbox](#choose-a-worker-sandbox). | | CONNECT returns 407 | Shared clients need an HTTPS proxy and one proxy bearer header. Check the token audience, expiry, broker boot, and separate service credential. | | CONNECT returns 403 or keeps asking for approval | Compare rule ID, version, port, grant scope, memberships and stop controls. A denied or expired prompt requires an explicit retry. Check that the Companion is paired, online and can access the selected identity. | +| CONNECT returns 502 | Read `Countersign-Error`, and find its `Countersign-Request-ID` in the Gateway's log line `Gateway CONNECT `. `proxy_denied`: the egress proxy refused the `CONNECT` with a 4xx answer. Check that its ACLs allow the site's hostname and port. `proxy_unavailable`: the Gateway could not use the egress proxy. It could not reach it, the proxy's name resolved only inside `denied_networks`, or the proxy closed the connection, never answered or answered with something that is not HTTP. Check `egress_proxy`, the proxy itself and the network to it. `upstream_failure`: the site could not be reached, or, on either route, the Gateway could not load the rule's `roots` or `revocation` files, so check those first. On a route through the egress proxy, the proxy answered 5xx because it could not reach the site: check the proxy's DNS and route to the site. On a direct route, check the site's DNS answers against its address policy, and the route to it. The Gateway never tries a proxied destination directly. | | CONNECT returns 429, 503 or 504 | Inspect limits per grant/user, Broker availability, worker fencing, clock synchronization and pending approval. Capture the safe `Countersign-Error` and `Countersign-Request-ID` response headers for correlation. | | CONNECT succeeds, then TLS closes | Check the Workload's current public proxy CA, exact hostname/SNI, server roots/status, issuer profile, client-certificate request and native provider. Check the owned status before treating a TLS failure as an application response. | | A long session closes | Check grant, certificate, status and token deadlines, lease renewal, host clock drift and revocation. Closing the Companion prevents new signatures. An established connection continues while its remote authority remains valid. | diff --git a/docs/reference/config.md b/docs/reference/config.md index 6dae8f1..a471035 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -111,7 +111,8 @@ This sidecar example assumes the Gateway's private Workload-facing address is `1 | `workload_token` | Sidecar only, required | Private file containing the backend-issued Workload bearer token. The Gateway rereads it on Broker calls. | | `workload` | Sidecar only, required | Object containing `integrator`, `user`, `context`, `workload` IDs and a positive `workload_generation`. | | `destinations` | Required | Array of Gateway entries. A sidecar needs at least one to construct its CA constraints. | -| `denied_networks` | `[]` | Additional IPv4 or IPv6 CIDRs to block across destinations. | +| `denied_networks` | `[]` | Additional IPv4 or IPv6 CIDRs to block: for every destination on a direct route, and for the egress proxy's own addresses. | +| `egress_proxy` | Omitted → every destination connects directly | The egress proxy: an `http` URL with a host and a port, such as `http://proxy.internal:3128`. Every destination goes through it, unless its rule says `"egress": "direct"`. The Gateway resolves the proxy's host through system DNS on every connection, drops any answer inside `denied_networks`, and sends a plain-text `CONNECT` with the site's name and port and no credentials. It resolves nothing for the site: the egress proxy does, and its ACLs are that site's address policy. An `https` URL, a URL with credentials, a path, a query or fragment, or a missing port fails loading. The Gateway never takes the egress proxy from a proxy environment variable such as `HTTPS_PROXY`, never tries a proxied destination directly, and always reaches the Broker directly. [Reach sites through an egress proxy](../operate.md#reach-sites-through-an-egress-proxy) | | `clock_uncertainty_ms` | `1000` | Allowed UTC difference from the Broker, in milliseconds, from 0 through 29999. The Gateway subtracts the margin from absolute deadlines and checks its local clock against monotonic elapsed time. | | `worker_sandbox` | `"namespaces"` | The worker sandbox for every grant worker, in sidecar and shared mode. `"namespaces"` is the namespace sandbox: bubblewrap gives each grant worker its own filesystem, process and network view, and the grant worker then confines itself with a seccomp allow-list, Landlock and non-dumpable memory. It needs `/usr/bin/bwrap` and a host that lets the Gateway create namespaces. `"filter"` is the filter sandbox: the same confinement without the namespaces, for a host that refuses them, such as ECS Fargate. It needs no bubblewrap. Any other value fails loading. The Gateway never switches from one to the other. [Choose a worker sandbox](../operate.md#choose-a-worker-sandbox) | | `shared` | Omitted → sidecar | Shared-mode object. Requires PEM `listener_cert` and `listener_key`, a `gateway_token` file holding the Broker's matching service secret, and a `ca` source below. Omit both sidecar Workload fields. | @@ -125,7 +126,8 @@ Use a unique stable `id`, positive `version`, and unique `pattern` at both servi | `whole_wildcard` | Broker | `false`. Set to `true` on a wildcard entry to offer a grant for the wildcard scope. | | `client_trust` | Broker | Optional object. `ca_file` selects alternative PEM client roots. `user_fingerprints` maps user IDs to allowed leaf SHA-256 fingerprints (standard padded base64 of 32 bytes). Omission permits identities passing the other checks. A supplied map requires a match for that user. `revocation` configures status imports. | | `approval` | Gateway | `"exact_host"`. Set to `"whole_wildcard"` only for a wildcard with the corresponding Broker setting. | -| `private_networks` | Gateway | `[]`. For an exact private destination, list allowed CIDRs. When present, every DNS answer must be within the configured prefixes and pass the address safety checks. | +| `private_networks` | Gateway | `[]`. For an exact private destination on a direct route, list allowed CIDRs. When present, every DNS answer must be within the configured prefixes and pass the address safety checks. With `egress_proxy` set, a rule that has `private_networks` and no `"egress": "direct"` fails loading, because nobody would check them: remove the field, or set `"egress": "direct"`. | +| `egress` | Gateway | Omitted → through `egress_proxy` when it is set, else direct. `"direct"` connects to the site directly, with the Gateway's own address policy, even when `egress_proxy` is set. Without `egress_proxy` it changes nothing. Any other value fails loading. | | `roots` | Gateway | Required PEM server trust anchors for this site's upstream certificate. | | `revocation` | Gateway | Optional strict status imports for the upstream server chain. | diff --git a/docs/specs/2026-11-17-countersign-design.md b/docs/specs/2026-11-17-countersign-design.md index d6062b9..7a1f12b 100644 --- a/docs/specs/2026-11-17-countersign-design.md +++ b/docs/specs/2026-11-17-countersign-design.md @@ -64,7 +64,7 @@ Exact entries take precedence over wildcard matches. Reject conflicting definiti The default Gateway admits HTTPS CONNECT to configured destinations. Unknown destinations receive a policy denial. Integrators can arrange ordinary egress separately. A protected failure always closes or denies the attempt. The Gateway must never convert it into an opaque pass-through tunnel. At a site claiming mandatory routing, the Integrator restricts direct network access including aliases, IP literals, fronted/shared endpoints and alternate proxies. Environment variables alone provide client configuration. -Resolve the canonical host in the trusted Gateway parent. Validate the answer set against the site's address policy, normalize IPv4-mapped IPv6 and fail on mixed permitted/forbidden answers. Block metadata, loopback, link-local and infrastructure addresses. Private sites require an explicit host/prefix/port rule. Connect to the selected IP and port without a second DNS resolution, retain the DNS name for TLS verification, and repeat policy and resolution checks on reconnect. Check CONNECT authority, downstream SNI and each HTTP Host or HTTP/2 :authority for agreement. Reject conflicting absolute-form URLs, connection upgrades to unrestricted tunnels and cross-origin coalescing. Redirects require the new destination's own policy decision. +A destination takes one of two routes. On a direct route, resolve the canonical host in the trusted Gateway parent. Validate the answer set against the site's address policy, normalize IPv4-mapped IPv6 and fail on mixed permitted/forbidden answers. Block metadata, loopback, link-local and infrastructure addresses. Private sites require an explicit host/prefix/port rule. Connect to the selected IP and port without a second DNS resolution, retain the DNS name for TLS verification, and repeat policy and resolution checks on reconnect. On a route through the operator's egress proxy (ADR 0003), the parent resolves only the egress proxy's name, sends a plain-text CONNECT with the canonical host and port, and resolves nothing for the site: the egress proxy resolves it and its ACLs carry the site's address policy. A proxied destination is never tried directly, and the Gateway still verifies the site's certificate against the DNS name before any client signature. Check CONNECT authority, downstream SNI and each HTTP Host or HTTP/2 :authority for agreement. Reject conflicting absolute-form URLs, connection upgrades to unrestricted tunnels and cross-origin coalescing. Redirects require the new destination's own policy decision. ## Gateway modes and issuance diff --git a/examples/demo-integrator/compose.squid.yaml b/examples/demo-integrator/compose.squid.yaml new file mode 100644 index 0000000..5a9bb68 --- /dev/null +++ b/examples/demo-integrator/compose.squid.yaml @@ -0,0 +1,44 @@ +# The opt-in Squid variant: compose_demo.py --squid layers this file over +# compose.yaml. The Gateway reaches the site only through a real Squid. +services: + site: + # The site leaves the backend network for site-host's, which only Squid + # shares. Stopping the site leaves its name and address behind, so Squid's + # connection is refused, as for a site whose server is down. + networks: !reset {} + network_mode: service:site-host + site-host: + init: true + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + user: "${CS_DEMO_UID:?run compose_demo.py}:${CS_DEMO_GID:?run compose_demo.py}" + image: countersign-demo/integrator:local + command: [sleep, infinity] + networks: + remote: + aliases: [site.test] + squid: + init: true + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + user: "${CS_DEMO_UID:?run compose_demo.py}:${CS_DEMO_GID:?run compose_demo.py}" + image: ubuntu/squid:6.6-24.04_edge@sha256:d9c34c5ab74c6dd41cf1070d9b4221aac072e67e983527ca2d41512958f77506 + # Squid in the foreground, without the image's log-tailing entrypoint. + entrypoint: [/usr/sbin/squid, -N, -f, /etc/squid/countersign/squid.conf] + tmpfs: + - /run/squid:rw,noexec,nosuid,size=64k,uid=${CS_DEMO_UID},gid=${CS_DEMO_GID},mode=0700 + - /tmp:rw,noexec,nosuid,size=16m,uid=${CS_DEMO_UID},gid=${CS_DEMO_GID},mode=0700 + volumes: + - ${CS_DEMO_STATE}/squid:/etc/squid/countersign:ro + - ${CS_DEMO_STATE}/squid-log:/var/log/squid + healthcheck: + test: [CMD, /usr/sbin/squid, -f, /etc/squid/countersign/squid.conf, -k, check] + interval: 1s + timeout: 2s + retries: 30 + networks: [backend, remote] + +networks: + remote: {internal: true} diff --git a/examples/demo-integrator/compose_demo.py b/examples/demo-integrator/compose_demo.py index 1da3b65..5bd3db2 100644 --- a/examples/demo-integrator/compose_demo.py +++ b/examples/demo-integrator/compose_demo.py @@ -6,6 +6,7 @@ import ipaddress import json import os +import re from pathlib import Path import secrets import shlex @@ -15,6 +16,7 @@ import sys import tempfile import threading +import time HERE = Path(__file__).resolve().parent ROOT = HERE.parents[1] @@ -46,16 +48,143 @@ def prepare_config(state, fixture, port): (state/"broker/broker.json").write_text(json.dumps(config)) -def gateway_config(state, address): +def gateway_config(state, address, egress_proxy=None): network = str(ipaddress.IPv4Network(address+"/32")) + destination = dict(**DESTINATION, private_networks=[network], roots="/config/ca.pem") + if egress_proxy: + # Squid pins the site to its address instead (squid.conf), as ADR 0003 has it. + del destination["private_networks"] config = dict(listen="@WORKLOAD_IP@:8080",proxy_url="http://gateway:8080",ca_output="/trust/sidecar.pem", broker_url="https://broker:9443",broker_ca="/config/ca.pem",workload_token="/credential/workload.jwt", workload=dict(integrator="demo",user="demo-user",context="demo-context", workload="demo-curl",workload_generation=1), - destinations=[dict(**DESTINATION, private_networks=[network], roots="/config/ca.pem")]) + destinations=[destination]) + if egress_proxy: + config["egress_proxy"] = egress_proxy (state/"gateway/gateway.json").write_text(json.dumps(config)) +def wait_for(condition, what, seconds=30): + """Wait for a condition the demo's services make true, such as a log line.""" + deadline=time.monotonic()+seconds + while not (found:=condition()): + if time.monotonic()>deadline: + raise RuntimeError("Timed out waiting for "+what) + threading.Event().wait(.2) + return found + + +class Squid: + """The Squid variant's egress proxy, as the launcher configures and observes it.""" + CONFIG="/etc/squid/countersign/squid.conf" + ALLOW="http_access allow CONNECT demo_site demo_port demo_prefixes\n" + REFUSE="http_access deny CONNECT demo_site\n" + + def __init__(self, compose, state): + self.compose,self.conf,self.logs=compose,state/"squid"/"squid.conf",state/"squid-log" + + def configure(self, address, allow=True): + site=ipaddress.IPv4Address(address) + # Every address but the site's, so a name with one answer elsewhere is refused. + outside=f"0.0.0.0-{site-1} {site+1}-255.255.255.255 ::/0" + text=((HERE/"squid.conf").read_text().replace("@SITE_PREFIXES@",str(ipaddress.IPv4Network(address+"/32"))) + .replace("@SITE_OUTSIDE@",outside)) + if text.count(self.ALLOW)!=1: + raise RuntimeError("squid.conf has no single rule that allows the site") + if not allow: + text=text.replace(self.ALLOW,self.REFUSE) + staged=self.conf.with_suffix(".next") + staged.write_text(text) + staged.replace(self.conf) + + def reload(self): + """Reconfigure the running Squid and wait until it accepts connections again.""" + log=self.logs/"cache.log" + start=log.stat().st_size + run(*self.compose,"exec","-T","squid","squid","-f",self.CONFIG,"-k","reconfigure") + def reopened(): + with log.open("rb") as file: + file.seek(start) + return b"Accepting HTTP Socket connections" in file.read() + wait_for(reopened,"Squid to reopen its port after reconfiguring") + + def version(self): + return run(*self.compose,"exec","-T","squid","squid","-v",capture_output=True,text=True).stdout.splitlines()[0] + + def gateway_logged(self, line, what): + def logged(): + return line in run(*self.compose,"logs","--no-log-prefix","gateway",capture_output=True,text=True).stdout + wait_for(logged,what,10) + + def no_direct_route(self, address): + """The Gateway leaves the site's address policy to Squid and has no other way to the site.""" + line="Gateway egress proxy squid:3128 carries the address policy of destinations: demo-site" + self.gateway_logged(line,"the Gateway's egress proxy startup line") + print(line,flush=True) + probe=lambda script: subprocess.run([*self.compose,"exec","-T","gateway","bash","-c",script,"probe",address], + timeout=30,capture_output=True).returncode + if probe("getent hosts site.test")!=2: + raise RuntimeError("The Gateway can resolve site.test without Squid") + # 1: refused or unreachable, 124: no answer. Anything else is not an answer. + if probe('timeout 3 bash -c "exec 3<>/dev/tcp/$1/9444"') not in (1,124): + raise RuntimeError(f"The Gateway can connect to the site at {address} without Squid") + print(f"The Gateway can neither resolve site.test nor connect to {address}:9444 directly",flush=True) + + def mark(self): + log=self.logs/"access.log" + return log.stat().st_size if log.exists() else 0 + + def entries(self, mark): + """Squid's access log lines since mark, as fields of its native format.""" + log=self.logs/"access.log" + if not log.exists(): return [] + with log.open("rb") as file: + file.seek(mark) + return [line.split() for line in file.read().decode().splitlines()] + + def logged(self, mark, action, hierarchy): + """Squid's one access log line since mark: a CONNECT to the site's hostname and port.""" + def connects(): + return [e for e in self.entries(mark) if len(e)>8 and e[5]=="CONNECT"] + found=wait_for(connects,"Squid's access log line for the Workload's CONNECT",10) + if len(found)!=1 or not re.fullmatch(action,found[0][3]) or found[0][6]!="site.test:9444" \ + or found[0][8]!=hierarchy: + raise RuntimeError(f"Squid did not log one {action} CONNECT site.test:9444 {hierarchy}: {found}") + return " ".join(found[0]) + + def fails(self, curl, error): + """The Workload's request fails at the Gateway with error, which logs it under the same request ID.""" + completed=subprocess.run(curl,timeout=110,capture_output=True,text=True) + if completed.returncode==0: + raise RuntimeError(f"The Workload's request succeeded where it should fail with {error}") + # curl dumps the head of the Gateway's answer to its CONNECT. + head=completed.stdout.replace("\r\n","\n").strip().split("\n\n")[-1].splitlines() + headers={key.lower():value for key,_,value in (line.partition(": ") for line in head[1:])} + request_id=headers.get("countersign-request-id") + if not head or not head[0].startswith("HTTP/1.1 502 ") or headers.get("countersign-error")!=error or not request_id: + raise RuntimeError(f"The Gateway did not answer 502 {error}: {completed.stdout!r} {completed.stderr!r}") + line=f"Gateway CONNECT {request_id} {error}" + self.gateway_logged(line,"the Gateway's log line "+line) + return f"{head[0]}, Countersign-Error: {error}, logged as {line!r}" + + def failures(self, curl, address): + """With the grant still live, each way the proxied leg can fail, and the Workload learns which.""" + self.configure(address,allow=False) + self.reload() + mark=self.mark() + print("Squid refused the site: "+self.fails(curl,"proxy_denied"),flush=True) + # HIER_NONE: Squid opened no connection to the site. + print("Squid logged the refusal: "+self.logged(mark,r"TCP_DENIED(_ABORTED)?/403","HIER_NONE/-"),flush=True) + self.configure(address) + self.reload() + run(*self.compose,"stop","site") + mark=self.mark() + print("Squid could not reach the stopped site: "+self.fails(curl,"upstream_failure"),flush=True) + print("Squid logged its 503: "+self.logged(mark,r"TCP_TUNNEL/503","HIER_DIRECT/"+address),flush=True) + run(*self.compose,"stop","squid") + print("With Squid stopped: "+self.fails(curl,"proxy_unavailable"),flush=True) + + def use_namespaces(stack, state, name): override=os.environ.get("CS_DEMO_GATEWAY_PROFILE") restriction=Path("/proc/sys/kernel/apparmor_restrict_unprivileged_userns") @@ -129,7 +258,7 @@ def record_job(state, url, completed): staged.replace(target) -def verify(desktop, browser, origin, fixture, curl, state, url): +def verify(desktop, browser, origin, fixture, curl, state, url, squid=None, address=None, failing=None): from playwright.sync_api import sync_playwright, expect with sync_playwright() as pw: session=pw.chromium.connect_over_cdp(desktop) @@ -149,6 +278,7 @@ def verify(desktop, browser, origin, fixture, curl, state, url): expect(tab.locator("#message")).to_contain_text("Confirm the account") desktop_page.get_by_role("button",name="Confirm pairing").click() expect(desktop_page.locator("#pairs")).to_contain_text("Connected") + mark=squid.mark() if squid else None with subprocess.Popen(curl, stdout=subprocess.PIPE,text=True) as workload: try: choices=desktop_page.locator("#requests .identity") @@ -164,6 +294,9 @@ def verify(desktop, browser, origin, fixture, curl, state, url): if workload.returncode or json.loads(output)["fingerprint"]!=fingerprint: raise RuntimeError("Upstream did not accept the selected native certificate") print("Compose mTLS accepted selected SHA-256 "+fingerprint,flush=True) + if squid: + print("Squid tunneled the Workload's CONNECT: "+squid.logged(mark,r"TCP_TUNNEL/200","HIER_DIRECT/"+address), + flush=True) # The Integrator's job page reads the result the launcher records. record_job(state,url,subprocess.CompletedProcess(curl,workload.returncode,output,"")) job=web.new_page() @@ -173,6 +306,8 @@ def verify(desktop, browser, origin, fixture, curl, state, url): expect(job.locator("#job-approved")).to_have_text(fingerprint) expect(job.locator("#match")).to_be_visible() print("Job page matched the fingerprint the site received with the approval",flush=True) + if squid: + squid.failures(failing,address) finally: if workload.poll() is None: workload.terminate() @@ -202,23 +337,39 @@ def session(args): CS_DEMO_PORT=str(args.port),CS_DEMO_GATEWAY_PROFILE=profile) prepare_config(state,fixture,args.port) compose=["docker","compose","-p",project,"-f",str(HERE/"compose.yaml")] + squid=None + if args.squid: + compose+=["-f",str(HERE/"compose.squid.yaml")] + for name in ("squid","squid-log"): + (state/name).mkdir(mode=0o700) + squid=Squid(compose,state) run(*compose,"config","-q") run(*compose,"build","broker","gateway","site",seconds=580) stack.callback(run,*compose,"down","--volumes","--remove-orphans",seconds=60) run(*compose,"up","--wait","--wait-timeout","55","broker","site","integrator",seconds=100) - site=run(*compose,"ps","-q","site",capture_output=True,text=True).stdout.strip() + # In the Squid variant the site has site-host's network and address. + site=run(*compose,"ps","-q","site-host" if squid else "site",capture_output=True,text=True).stdout.strip() address=run("docker","inspect","-f","{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", site,capture_output=True,text=True).stdout.strip() - gateway_config(state,address) + if squid: + squid.configure(address) + run(*compose,"up","--wait","--wait-timeout","55","squid",seconds=100) + print("Egress proxy: "+squid.version(),flush=True) + gateway_config(state,address,"http://squid:3128" if squid else None) run(*compose,"up","--wait","--wait-timeout","35","gateway") # The Workload can read the newly generated public trust anchor and has # no credential/control-plane mount or Gateway process in its namespace. gateway_id=run(*compose,"ps","-q","gateway",capture_output=True,text=True).stdout.strip() gateway_pid=run("docker","inspect","-f","{{.State.Pid}}",gateway_id,capture_output=True,text=True).stdout.strip() if not gateway_pid.isdecimal() or int(gateway_pid)<=1: raise RuntimeError("invalid Gateway host PID") + if squid: + squid.no_direct_route(address) + # In the Squid variant the site cannot even resolve the Gateway's name: + # Docker's DNS on an internal network fails a name it does not hold. + refused="['ENOTFOUND','EAI_AGAIN'].includes(e.code)" if squid else "e.code==='ECONNREFUSED'" run(*compose,"exec","-T","site","node","-e", "const s=require('node:net').connect(8080,'gateway');s.on('connect',()=>process.exit(1));" - "s.setTimeout(3000,()=>process.exit(2));s.on('error',e=>process.exit(e.code==='ECONNREFUSED'?0:3));") + "s.setTimeout(3000,()=>process.exit(2));s.on('error',e=>process.exit("+refused+"?0:3));") run(*compose,"run","--rm","-T","--entrypoint","sh","workload","-ec", 'test "$(id -u)" = 65534 && test -r /trust/sidecar.pem && test ! -e /credential && test ! -e /config && ' 'test ! -r "/proc/$1/mem" && test ! -r "/proc/$1/fd" && ' @@ -249,7 +400,10 @@ def start(command): url="https://site.test:9444/" curl=[*compose,"run","--rm","-T","workload","--fail","--silent","--show-error","--max-time","95",url] if args.verify: - verify(desktop_endpoint,browser_endpoint,origin,fixture,curl,state,url) + # The head of the Gateway's answer to the Workload's CONNECT, when the proxied leg fails. + failing=[*compose,"run","--rm","-T","workload","--silent","--show-error","--max-time","95", + "--dump-header","-","--output","/dev/null",url] if squid else None + verify(desktop_endpoint,browser_endpoint,origin,fixture,curl,state,url,squid,address,failing) return print(f"In the Companion's Integrators tab, enter {origin} and choose Pair. Sign in to the demo tab using the secret from {fixture/'login.secret'}.") print("Enter the Companion pairing code in the browser and confirm the pairing in the Companion.") @@ -275,6 +429,8 @@ def main(): parser.add_argument("--browser",type=Path,help="Chromium executable (defaults to system or Playwright Chromium)") parser.add_argument("--port",type=int,default=8443,help="loopback HTTPS port (default 8443)") parser.add_argument("--verify",action="store_true",help="drive the demo's actual browser and Companion UIs") + parser.add_argument("--squid",action="store_true", + help="reach the site through a real Squid egress proxy (a maintainer's qualification)") parser.add_argument("--session",action="store_true",help=argparse.SUPPRESS) args=parser.parse_args() if not 1024<=args.port<=65535: parser.error("choose an unprivileged port from 1024 to 65535") diff --git a/examples/demo-integrator/squid.conf b/examples/demo-integrator/squid.conf new file mode 100644 index 0000000..ef02914 --- /dev/null +++ b/examples/demo-integrator/squid.conf @@ -0,0 +1,31 @@ +# The egress proxy of the Compose demo's Squid variant (compose_demo.py --squid). +# The launcher writes this file to the demo's state with the site's address in +# place of @SITE_PREFIXES@, and every other address in place of @SITE_OUTSIDE@. +# Plain HTTP CONNECT, no TLS interception, no cache. +http_port 3128 +pid_filename /run/squid/squid.pid +access_log stdio:/var/log/squid/access.log +cache_log /var/log/squid/cache.log +logfile_rotate 0 +cache deny all +coredump_dir /tmp +pinger_enable off +shutdown_lifetime 1 seconds + +acl CONNECT method CONNECT +acl demo_site dstdomain site.test +acl demo_port port 9444 +acl to_metadata dst 169.254.169.254 fd00:ec2::254 168.63.129.16 +acl to_link_local dst 169.254.0.0/16 fe80::/10 +acl to_loopback dst 127.0.0.0/8 ::1 +acl demo_prefixes dst @SITE_PREFIXES@ +# A dst ACL matches when any one of a name's addresses is in its list. +acl demo_outside dst @SITE_OUTSIDE@ + +http_access deny !CONNECT +http_access deny to_metadata +http_access deny to_link_local +http_access deny to_loopback +http_access deny demo_site demo_outside +http_access allow CONNECT demo_site demo_port demo_prefixes +http_access deny all