From d0fb09fc7e996b980eab2d61cf6910ba764383ff Mon Sep 17 00:00:00 2001 From: bitgobot Date: Wed, 21 Oct 2026 16:14:43 +0000 Subject: [PATCH] fix(express): rebuild image to drop eMudhra TLS trust bitgo/express:17.6.0 validates TLS against Node's embedded root store, which still trusts the eMudhra roots compromised in the October 2026 CA incident, and ships no system CA bundle, so TLS-sensitive workloads behind Express accept eMudhra-rooted chains. - Move the base to node:24.15.0-bookworm-slim (newest available base; .nvmrc already tracks Node 24). - Install ca-certificates so a usable system bundle exists, and strip the three eMudhra roots Debian 12 still ships until the distro removes them. - Set NODE_OPTIONS=--use-openssl-ca and SSL_CERT_FILE so outbound TLS validates against that bundle instead of Node's embedded roots. - Add scripts/verify-express-image-trust.sh and run it in the CI docker-build job. It asserts the bundle exists and is free of eMudhra trust and that TLS verification follows the system store. Corrected Node trust data remains outstanding: v24.15.0's embedded root store still lists emSign Root CA - G1/C1 and emSign ECC Root CA - G3/C3. --use-openssl-ca keeps that store out of TLS verification, but a base bump to a Node release with corrected trust data is still needed once a suitable tag is available. Ticket: COINS-1861 Session-Id: f02afc86-4d81-4a81-99b8-96565f3c39ac Task-Id: e88e5f0c-081b-4523-b980-8235f9aefc3a Requested-By: Otto Allmendinger --- .github/workflows/ci.yml | 3 + Dockerfile | 24 +++++- modules/express/README.md | 12 +++ scripts/verify-express-image-trust.sh | 114 ++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 4 deletions(-) create mode 100755 scripts/verify-express-image-trust.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93fffa30f01..d2c91656544 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -382,6 +382,9 @@ jobs: GIT_HASH=${{ github.sha }} SOCKET_SECURITY_MODE=${{ env.SOCKET_SECURITY_MODE }} + - name: Verify CA trust configuration + run: ./scripts/verify-express-image-trust.sh bitgo/express:${{ github.sha }} + - name: Test Express Docker image id: docker-test run: | diff --git a/Dockerfile b/Dockerfile index 38020d5d540..667d57a0369 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # An elaborated scheme to build all the dependencies of all packages first in a cached layer # https://stackoverflow.com/a/63142468/134409 # https://medium.com/@emilefugulin/building-a-sane-docker-image-for-typescript-lerna-and-prisma-2-76d8ff9926e4 -FROM node:22.22.0-bookworm-slim@sha256:f86be15afa9a8277608e141ce2a8aa55d3d9c40845921b8511f4fb7897be2554 AS filter-packages-json +FROM node:24.15.0-bookworm-slim@sha256:4e6b70dd6cbfc88c8157ba19aa3d9f9cce6ba4703576d55459e45efcbc9c5f5d AS filter-packages-json LABEL maintainer="Developer Relations " COPY package.json yarn.lock lerna.json ./ @@ -12,7 +12,7 @@ COPY modules ./modules # delete all the non package.json files under `./modules/` RUN find modules \! -name "package.json" -mindepth 2 -maxdepth 2 -print | xargs rm -rf -FROM node:22.22.0-bookworm-slim@sha256:f86be15afa9a8277608e141ce2a8aa55d3d9c40845921b8511f4fb7897be2554 AS builder +FROM node:24.15.0-bookworm-slim@sha256:4e6b70dd6cbfc88c8157ba19aa3d9f9cce6ba4703576d55459e45efcbc9c5f5d AS builder ARG SOCKET_SECURITY_MODE=monitor ENV SOCKET_SECURITY_MODE=${SOCKET_SECURITY_MODE} RUN apt-get update && apt-get install -y git python3 make g++ libtool autoconf automake @@ -34,8 +34,20 @@ RUN \ rm -r modules/*/src -FROM node:22.22.0-bookworm-slim@sha256:f86be15afa9a8277608e141ce2a8aa55d3d9c40845921b8511f4fb7897be2554 -RUN apt-get update && apt-get install -y tini +FROM node:24.15.0-bookworm-slim@sha256:4e6b70dd6cbfc88c8157ba19aa3d9f9cce6ba4703576d55459e45efcbc9c5f5d +# The runtime validates TLS against the system CA bundle (NODE_OPTIONS at the +# bottom), so ca-certificates must be installed. Debian's package still trusts +# eMudhra roots after the October 2026 CA compromise, so they are stripped here +# until the distro removes them; scripts/verify-express-image-trust.sh asserts +# that none remain. +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tini && rm -rf /var/lib/apt/lists/* && \ + for f in $(find /usr/share/ca-certificates -name '*.crt'); do \ + if openssl x509 -in "$f" -noout -subject 2>/dev/null | grep -qi 'emudhra\|emsign'; then \ + echo "removing compromised trust anchor: $f" && rm "$f"; \ + fi; \ + done && \ + sed -i -e '/emSign/d' -e '/emudhra/Id' /etc/ca-certificates.conf && \ + update-ca-certificates --fresh # copy the root node_modules to the bitgo-express parent node_modules COPY --from=builder /tmp/bitgo/node_modules /var/node_modules/ COPY --from=builder /tmp/bitgo/modules/express /var/bitgo-express/ @@ -374,5 +386,9 @@ LABEL git_hash=${GIT_HASH} USER node ENV NODE_ENV=production ENV BITGO_BIND=0.0.0.0 +# Validate outbound TLS against the system CA bundle instead of Node's embedded +# root store, so OS trust updates (e.g. revoked roots) take effect. +ENV NODE_OPTIONS=--use-openssl-ca +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt EXPOSE 3080 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/node", "/var/bitgo-express/bin/bitgo-express"] diff --git a/modules/express/README.md b/modules/express/README.md index 12273b7af05..c10f49c1a22 100644 --- a/modules/express/README.md +++ b/modules/express/README.md @@ -97,6 +97,18 @@ Environment: prod Base URI: https://0.0.0.0:4000 ``` +### TLS trust + +The BitGo Express image validates outbound TLS against the system CA +store (`/etc/ssl/certs/ca-certificates.crt`, from the Debian +`ca-certificates` package) instead of Node's embedded root store. The +container sets `NODE_OPTIONS=--use-openssl-ca` and +`SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt`. To trust an +additional CA (for example an internal CA), set `NODE_EXTRA_CA_CERTS` +to a PEM file that contains it — extra certificates are added on top +of the configured system store — or point `SSL_CERT_FILE` at a bundle +that includes it (this replaces the default bundle). + ### Building the Docker container To build the BitGo Express Docker container yourself from the source code, check out the latest **master** branch and run `docker build` from the project root. diff --git a/scripts/verify-express-image-trust.sh b/scripts/verify-express-image-trust.sh new file mode 100755 index 00000000000..600b12bb402 --- /dev/null +++ b/scripts/verify-express-image-trust.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# Verify a BitGo Express image cannot validate TLS chains rooted in eMudhra +# (October 2026 CA compromise): the system CA bundle carries no eMudhra/emSign +# trust, and TLS verification follows that bundle rather than Node's embedded +# root store. +# +# Usage: scripts/verify-express-image-trust.sh + +set -euo pipefail + +IMAGE="${1:?usage: verify-express-image-trust.sh }" + +WORK_DIR=$(mktemp -d) +trap 'rm -rf "$WORK_DIR"' EXIT + +echo "Verifying CA trust configuration of $IMAGE" + +# The system CA bundle must exist and contain certificates. +if ! docker run --rm --entrypoint /bin/sh "$IMAGE" -c \ + 'test -s /etc/ssl/certs/ca-certificates.crt && grep -q "BEGIN CERTIFICATE" /etc/ssl/certs/ca-certificates.crt'; then + echo "::error::missing or empty /etc/ssl/certs/ca-certificates.crt in $IMAGE" + exit 1 +fi + +# The system bundle must carry no eMudhra/emSign trust. Node's embedded root +# store is reported but only matters when the runtime consults it, which the +# probes below assert it does not. +docker run --rm --entrypoint /usr/local/bin/node "$IMAGE" -e ' + const fs = require("fs"); + const tls = require("tls"); + const { X509Certificate } = require("crypto"); + const pem = fs.readFileSync("/etc/ssl/certs/ca-certificates.crt", "utf8"); + const blocks = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g) || []; + const compromised = (x509) => /emudhra|emsign/i.test(`${x509.subject}\n${x509.issuer}`); + const embedded = tls.rootCertificates.map((block) => new X509Certificate(block)); + console.log(`system bundle certs: ${blocks.length}, embedded roots: ${embedded.length}`); + const hits = blocks.map((block) => new X509Certificate(block)).filter(compromised); + if (hits.length > 0) { + console.error(`eMudhra/emSign trust found in the system bundle:\n${hits.map((x509) => x509.subject).join("\n")}`); + process.exit(1); + } + const embeddedHits = embedded.filter(compromised); + if (embeddedHits.length > 0) { + console.log(`note: Node embedded root store lists ${embeddedHits.length} eMudhra/emSign root(s); NODE_OPTIONS=--use-openssl-ca keeps them out of TLS verification (asserted by the probes below)`); + } +' + +# TLS verification must follow the configured system bundle. A server whose +# chain is anchored in a throwaway probe CA is accepted only when that CA is in +# the bundle Node validates against; Node's embedded roots alone reject it. +openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes \ + -keyout "$WORK_DIR/probe-ca.key" -out "$WORK_DIR/probe-ca.crt" -days 1 \ + -subj "/CN=Express Trust Probe CA" +openssl req -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes \ + -keyout "$WORK_DIR/probe-leaf.key" -out "$WORK_DIR/probe-leaf.csr" \ + -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" +openssl x509 -req -in "$WORK_DIR/probe-leaf.csr" \ + -CA "$WORK_DIR/probe-ca.crt" -CAkey "$WORK_DIR/probe-ca.key" -CAcreateserial \ + -out "$WORK_DIR/probe-leaf.crt" -days 1 -copy_extensions copy + +PROBE_JS=' +const fs = require("fs"); +const tls = require("tls"); +if (process.env.PROBE_CA) { + fs.writeFileSync("/tmp/probe-bundle.pem", process.env.PROBE_CA); +} +const server = tls.createServer( + { key: process.env.PROBE_KEY, cert: process.env.PROBE_LEAF }, + (socket) => socket.end("ok") +); +server.listen(0, "127.0.0.1", () => { + const socket = tls.connect( + { host: "127.0.0.1", port: server.address().port, servername: "localhost", rejectUnauthorized: true }, + () => { + console.log("ACCEPTED"); + socket.end(); + server.close(); + } + ); + socket.on("error", (err) => { + console.log(`REJECTED ${err.code || err.message}`); + server.close(); + process.exitCode = 2; + }); +}); +' + +LEAF_KEY=$(cat "$WORK_DIR/probe-leaf.key") +LEAF_CERT=$(cat "$WORK_DIR/probe-leaf.crt") +PROBE_CA=$(cat "$WORK_DIR/probe-ca.crt") + +echo "Probe CA present in configured bundle -> chain must verify:" +printf '%s' "$PROBE_JS" | docker run -i --rm \ + -e PROBE_KEY="$LEAF_KEY" -e PROBE_LEAF="$LEAF_CERT" -e PROBE_CA="$PROBE_CA" \ + -e SSL_CERT_FILE=/tmp/probe-bundle.pem \ + --entrypoint /usr/local/bin/node "$IMAGE" - > "$WORK_DIR/accept.log" 2>&1 || true +cat "$WORK_DIR/accept.log" +if ! grep -q '^ACCEPTED' "$WORK_DIR/accept.log"; then + echo "::error::TLS verification does not follow the system CA store in $IMAGE" + exit 1 +fi + +echo "Probe CA absent from configured bundle -> chain must be rejected:" +printf '%s' "$PROBE_JS" | docker run -i --rm \ + -e PROBE_KEY="$LEAF_KEY" -e PROBE_LEAF="$LEAF_CERT" \ + --entrypoint /usr/local/bin/node "$IMAGE" - > "$WORK_DIR/reject.log" 2>&1 || true +cat "$WORK_DIR/reject.log" +if ! grep -q '^REJECTED' "$WORK_DIR/reject.log"; then + echo "::error::untrusted probe CA was accepted via the default bundle in $IMAGE" + exit 1 +fi + +echo "✅ CA trust verification passed for $IMAGE" \ No newline at end of file