From 4653a6688fb480836d39494649dd14cd1d912a1a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:46:27 +0800 Subject: [PATCH 01/28] store: FetchFile materializes one file of a record A reader of an archived claim needs its disk and sidecar, not the guest memory image Fetch pulls with them. --- sandboxd/store/dir/dir.go | 15 +++++++ sandboxd/store/s3/s3.go | 48 +++++++++++++++++++++ sandboxd/store/s3/s3_test.go | 62 +++++++++++++++++++++++++++ sandboxd/store/store.go | 2 + sandboxd/store/storetest/storetest.go | 14 ++++++ 5 files changed, 141 insertions(+) diff --git a/sandboxd/store/dir/dir.go b/sandboxd/store/dir/dir.go index 1dc40dc5..5eccef38 100644 --- a/sandboxd/store/dir/dir.go +++ b/sandboxd/store/dir/dir.go @@ -90,6 +90,21 @@ func (d *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e return dir, meta, digest, nil } +func (d *Store) FetchFile(ctx context.Context, id, rel string) (string, error) { + meta, err := d.ReadMeta(ctx, id) + if err != nil { + return "", err + } + path := filepath.Join(d.root, id, store.ExportGen(meta), rel) + switch _, statErr := os.Stat(path); { + case errors.Is(statErr, fs.ErrNotExist): + return "", store.ErrNotFound + case statErr != nil: + return "", statErr + } + return path, nil +} + func (d *Store) ReadMeta(_ context.Context, id string) ([]byte, error) { raw, err := os.ReadFile(filepath.Join(d.root, id, store.MetaFile)) //nolint:gosec // id pinned by the instance idRe before any call if errors.Is(err, fs.ErrNotExist) { diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 0cb1848c..5f49d3bc 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -154,6 +154,31 @@ func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e return export, meta, digest, nil } +func (s *Store) FetchFile(ctx context.Context, id, rel string) (string, error) { + meta, _, err := s.readMeta(ctx, id) + if err != nil { + return "", err + } + path := filepath.Join(s.staging, "cache", id, store.ExportGenHash(meta)+"-"+strings.ReplaceAll(rel, "/", "_")) + if _, statErr := os.Stat(path); statErr == nil { + return path, nil + } + flight := s.fetches.DoChan(path, func() (any, error) { + fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), transferBudget) + defer cancel() + return nil, s.populateFile(fctx, id, store.ExportGen(meta), rel, path) + }) + select { + case res := <-flight: + if res.Err != nil { + return "", res.Err + } + case <-ctx.Done(): + return "", ctx.Err() + } + return path, nil +} + func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { meta, _, err := s.readMeta(ctx, id) return meta, err @@ -414,6 +439,29 @@ func (s *Store) materialize(ctx context.Context, man manifest, root string) (err return g.Wait() } +// populateFile materializes one manifest file through the chunk path of a full fetch, so holes stay holes. +func (s *Store) populateFile(ctx context.Context, id, gen, rel, path string) error { + if _, err := os.Stat(path); err == nil { + return nil + } + man, err := s.readManifest(ctx, id, gen) + if err != nil { + return err + } + i := slices.IndexFunc(man.Files, func(f manifestFile) bool { return f.Path == rel }) + if i < 0 { + return store.ErrNotFound + } + part := path + ".part" + defer func() { _ = os.Remove(part) }() + one := manifest{Lineage: man.Lineage, Files: []manifestFile{man.Files[i]}} + one.Files[0].Path = filepath.Base(part) + if err = s.materialize(ctx, one, filepath.Dir(part)); err != nil { + return err + } + return os.Rename(part, path) +} + func (s *Store) fetchChunk(ctx context.Context, key string, f *os.File, size int64, chunk manifestChunk) error { out, err := s.get(ctx, key) if errors.Is(err, store.ErrNotFound) { diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index 7f7bb7af..4de71b46 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -718,6 +718,68 @@ func TestTransferBudgetStaysBelowTheReuseWindow(t *testing.T) { } } +func TestFetchFileDownloadsOnlyTheRequestedFile(t *testing.T) { + const id = "ck_00000000000000aa" + meta := []byte(`{"id":"` + id + `"}`) + fake := &fakeS3{objects: map[string][]byte{}} + st := newTestStore(t, fake) + staging := stageRecord(t, st, id, meta, "disk-bytes") + if err := os.WriteFile(filepath.Join(staging, store.ExportDir, "memory.bin"), []byte("memory-bytes"), 0o600); err != nil { + t.Fatalf("write memory: %v", err) + } + if err := st.Publish(t.Context(), staging, id, ""); err != nil { + t.Fatalf("publish: %v", err) + } + fake.resetRequests() + + path, err := st.FetchFile(t.Context(), id, "disk.img") + if err != nil { + t.Fatalf("FetchFile: %v", err) + } + if got, readErr := os.ReadFile(path); readErr != nil || string(got) != "disk-bytes" { + t.Fatalf("fetched file: %q, %v", got, readErr) + } + memory := sha256.Sum256([]byte("memory-bytes")) + memoryKey := "ck/chunks/" + id + "/" + hex.EncodeToString(memory[:]) + if n := fake.requestCount(http.MethodGet, memoryKey); n != 0 { + t.Errorf("GET %s = %d requests, want 0", memoryKey, n) + } + disk := sha256.Sum256([]byte("disk-bytes")) + diskKey := "ck/chunks/" + id + "/" + hex.EncodeToString(disk[:]) + if again, err := st.FetchFile(t.Context(), id, "disk.img"); err != nil || again != path { + t.Fatalf("second FetchFile = %q, %v, want the cached %q", again, err, path) + } + if n := fake.requestCount(http.MethodGet, diskKey); n != 1 { + t.Errorf("GET %s = %d requests, want 1 across two fetches", diskKey, n) + } +} + +func TestFetchFileInstallsOnlyACompleteDownload(t *testing.T) { + const id = "ck_00000000000000aa" + meta := []byte(`{"id":"` + id + `"}`) + fake := &fakeS3{objects: map[string][]byte{}, exportGate: make(chan struct{}), exportEntered: make(chan struct{}, 1)} + st := newTestStore(t, fake) + publishRecord(t, st, id, meta, "disk-bytes") + cached := filepath.Join(st.staging, "cache", id, store.ExportGenHash(meta)+"-disk.img") + + fetched := make(chan error, 1) + go func() { + _, err := st.FetchFile(t.Context(), id, "disk.img") + fetched <- err + }() + <-fake.exportEntered + if _, err := os.Stat(cached); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a download in flight is already visible at the cache path: %v", err) + } + close(fake.exportGate) + if err := <-fetched; err != nil { + t.Fatalf("FetchFile: %v", err) + } + if got, err := os.ReadFile(cached); err != nil || string(got) != "disk-bytes" { + t.Errorf("cached file %q, %v, want the object's bytes", got, err) + } +} + func publishRecord(t *testing.T, st *Store, id string, meta []byte, content string) { t.Helper() staging := stageRecord(t, st, id, meta, content) diff --git a/sandboxd/store/store.go b/sandboxd/store/store.go index 55227273..62f43872 100644 --- a/sandboxd/store/store.go +++ b/sandboxd/store/store.go @@ -65,6 +65,8 @@ type Store interface { PublishDigested(ctx context.Context, staging, id, lineage string) (string, error) // Fetch materializes a record's export locally; the caller holds the record lock until the clone ends. Fetch(ctx context.Context, id string) (dir string, meta []byte, digest string, err error) + // FetchFile materializes one file of a record's export locally, read-only to the caller, without fetching the rest of the record. + FetchFile(ctx context.Context, id, rel string) (string, error) // ReadMeta returns a record's metadata, or an error when the record does not exist. ReadMeta(ctx context.Context, id string) ([]byte, error) // Metas lists the metadata and digest of every record in this instance's id namespace. diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index cb7d63a1..4595a070 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -77,9 +77,23 @@ func RunContract(t *testing.T, st store.Store) { t.Errorf("second-generation export: %q, %v", got, readErr) } + filePath, err := st.FetchFile(ctx, id, "disk2.img") + if err != nil { + t.Fatalf("FetchFile: %v", err) + } + if got, readErr := os.ReadFile(filePath); readErr != nil || string(got) != "second-gen" { //nolint:gosec // test path + t.Errorf("fetched file: %q, %v", got, readErr) + } + if _, err = st.FetchFile(ctx, id, "absent.img"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("FetchFile of an absent file: %v, want store.ErrNotFound", err) + } + if err = st.Delete(ctx, id); err != nil { t.Fatalf("Delete: %v", err) } + if _, err = st.FetchFile(ctx, id, "disk2.img"); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("FetchFile after Delete: %v, want store.ErrNotFound", err) + } if _, err = st.ReadMeta(ctx, id); !errors.Is(err, store.ErrNotFound) { t.Fatalf("ReadMeta after Delete: %v, want store.ErrNotFound", err) } From da26a62775a426afcd4779f60c87b5b6fdcc0009 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:46:27 +0800 Subject: [PATCH 02/28] utils: CloneFile reflinks a file, falling back to a hole-skipping copy --- sandboxd/utils/clone.go | 31 ++++++++++++ sandboxd/utils/clone_linux.go | 42 ++++++++++++++++ sandboxd/utils/clone_other.go | 16 ++++++ sandboxd/utils/clone_test.go | 95 +++++++++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+) create mode 100644 sandboxd/utils/clone.go create mode 100644 sandboxd/utils/clone_linux.go create mode 100644 sandboxd/utils/clone_other.go create mode 100644 sandboxd/utils/clone_test.go diff --git a/sandboxd/utils/clone.go b/sandboxd/utils/clone.go new file mode 100644 index 00000000..7510a573 --- /dev/null +++ b/sandboxd/utils/clone.go @@ -0,0 +1,31 @@ +package utils + +import ( + "fmt" + "os" +) + +// CloneFile writes a new file dst that holds src's bytes, sharing extents where the filesystem can and copying only data ranges where it cannot. +func CloneFile(src, dst string) (err error) { + in, err := os.Open(src) //nolint:gosec // caller-owned paths + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) //nolint:gosec // caller-owned paths + if err != nil { + return err + } + defer func() { + if err != nil { + _ = os.Remove(dst) + } + }() + if err = reflink(out, in); err != nil { + if err = copyData(out, in); err != nil { + _ = out.Close() + return fmt.Errorf("copy %s: %w", src, err) + } + } + return out.Close() +} diff --git a/sandboxd/utils/clone_linux.go b/sandboxd/utils/clone_linux.go new file mode 100644 index 00000000..408975b6 --- /dev/null +++ b/sandboxd/utils/clone_linux.go @@ -0,0 +1,42 @@ +package utils + +import ( + "errors" + "io" + "os" + + "golang.org/x/sys/unix" +) + +func reflink(dst, src *os.File) error { + return unix.IoctlFileClone(int(dst.Fd()), int(src.Fd())) +} + +func copyData(dst, src *os.File) error { + info, err := src.Stat() + if err != nil { + return err + } + size := info.Size() + if err = dst.Truncate(size); err != nil { + return err + } + for off := int64(0); off < size; { + data, err := src.Seek(off, unix.SEEK_DATA) + if errors.Is(err, unix.ENXIO) { + return nil + } + if err != nil { + return err + } + hole, err := src.Seek(data, unix.SEEK_HOLE) + if err != nil { + return err + } + if _, err = io.Copy(io.NewOffsetWriter(dst, data), io.NewSectionReader(src, data, hole-data)); err != nil { + return err + } + off = hole + } + return nil +} diff --git a/sandboxd/utils/clone_other.go b/sandboxd/utils/clone_other.go new file mode 100644 index 00000000..fafc5752 --- /dev/null +++ b/sandboxd/utils/clone_other.go @@ -0,0 +1,16 @@ +//go:build !linux + +package utils + +import ( + "errors" + "io" + "os" +) + +func reflink(_, _ *os.File) error { return errors.ErrUnsupported } + +func copyData(dst, src *os.File) error { + _, err := io.Copy(dst, src) + return err +} diff --git a/sandboxd/utils/clone_test.go b/sandboxd/utils/clone_test.go new file mode 100644 index 00000000..8c46c333 --- /dev/null +++ b/sandboxd/utils/clone_test.go @@ -0,0 +1,95 @@ +package utils + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestCloneFileCopiesContentAndSize(t *testing.T) { + src, dst := writeSparse(t) + if err := CloneFile(src, dst); err != nil { + t.Fatalf("CloneFile: %v", err) + } + assertSameFile(t, src, dst) +} + +func TestCloneFileRefusesExistingDestination(t *testing.T) { + src, dst := writeSparse(t) + if err := os.WriteFile(dst, []byte("keep"), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + if err := CloneFile(src, dst); err == nil { + t.Fatal("CloneFile overwrote an existing destination") + } + if got, _ := os.ReadFile(dst); string(got) != "keep" { + t.Errorf("destination holds %q after the refusal, want it untouched", got) + } +} + +func TestCloneFileRemovesFailedDestination(t *testing.T) { + dst := filepath.Join(t.TempDir(), "dst") + if err := CloneFile(t.TempDir(), dst); err == nil { + t.Fatal("CloneFile copied a directory") + } + if _, err := os.Stat(dst); err == nil { + t.Error("a failed clone left its destination behind") + } +} + +func TestCopyDataCopiesDataRangesOnly(t *testing.T) { + srcPath, dstPath := writeSparse(t) + src, err := os.Open(srcPath) + if err != nil { + t.Fatalf("open src: %v", err) + } + defer func() { _ = src.Close() }() + dst, err := os.Create(dstPath) + if err != nil { + t.Fatalf("create dst: %v", err) + } + defer func() { _ = dst.Close() }() + if err = copyData(dst, src); err != nil { + t.Fatalf("copyData: %v", err) + } + assertSameFile(t, srcPath, dstPath) +} + +func writeSparse(t *testing.T) (src, dst string) { + t.Helper() + dir := t.TempDir() + src = filepath.Join(dir, "src") + f, err := os.Create(src) + if err != nil { + t.Fatalf("setup: %v", err) + } + if _, err = f.WriteAt([]byte("head"), 0); err != nil { + t.Fatalf("setup: %v", err) + } + if _, err = f.WriteAt([]byte("tail"), 8<<20); err != nil { + t.Fatalf("setup: %v", err) + } + if err = f.Truncate(16 << 20); err != nil { + t.Fatalf("setup: %v", err) + } + if err = f.Close(); err != nil { + t.Fatalf("setup: %v", err) + } + return src, filepath.Join(dir, "dst") +} + +func assertSameFile(t *testing.T, a, b string) { + t.Helper() + got, err := os.ReadFile(b) + if err != nil { + t.Fatalf("read copy: %v", err) + } + want, err := os.ReadFile(a) + if err != nil { + t.Fatalf("read source: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("copy differs from the source (%d vs %d bytes)", len(got), len(want)) + } +} From 150b9ebc3a08ce4709ef411b6bb2bd24e246ea0d Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:46:27 +0800 Subject: [PATCH 03/28] wire: RequestOp reads a request frame's op from its head --- protocol/wire/frame.go | 9 +++++++++ protocol/wire/frame_test.go | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/protocol/wire/frame.go b/protocol/wire/frame.go index 16a5f8ed..f06d8bd2 100644 --- a/protocol/wire/frame.go +++ b/protocol/wire/frame.go @@ -723,6 +723,15 @@ func AppendBulkRequest(buf []byte, op string, data []byte) []byte { return append(buf, '"', '}', '\n') } +// RequestOp returns the op tag of a request line, or "" when the line carries none; the line's head is enough. +func RequestOp(line []byte) string { + op, err := frameTag(line, reqTagHead, "op") + if err != nil { + return "" + } + return string(op) +} + // IsContinuation reports whether a request line feeds the RPC in flight instead of opening one. func IsContinuation(line []byte) bool { op, err := frameTag(line, reqTagHead, "op") diff --git a/protocol/wire/frame_test.go b/protocol/wire/frame_test.go index 6eb316cd..bcf2c473 100644 --- a/protocol/wire/frame_test.go +++ b/protocol/wire/frame_test.go @@ -413,6 +413,23 @@ func TestIsContinuation(t *testing.T) { } } +func TestRequestOp(t *testing.T) { + for _, tt := range []struct { + line string + want string + }{ + {`{"v":1,"op":"fs_read","path":"/etc/hostname"}`, FsRead{}.Op()}, + {`{"op":"fs_stat","v":1,"path":"/"}`, FsStat{}.Op()}, + {`{"v":1,"op":"info"`, Info{}.Op()}, + {`{"v":1,"path":"/"}`, ""}, + {"", ""}, + } { + if got := RequestOp([]byte(tt.line)); got != tt.want { + t.Errorf("RequestOp(%q) = %q, want %q", tt.line, got, tt.want) + } + } +} + func jsonEqual(t *testing.T, a, b []byte) bool { t.Helper() var av, bv any From 7cb464dfc21f86a5f037af3a95ab1647c5e2ce59 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:48:28 +0800 Subject: [PATCH 04/28] engine: StartReader mounts a claim's disks as a read-only view served by a chrooted silkd --- sandboxd/engine/engine.go | 88 ++++++++++++++------------- sandboxd/engine/engine_test.go | 2 +- sandboxd/engine/installca_test.go | 28 ++++++--- sandboxd/engine/reader.go | 84 ++++++++++++++++++++++++++ sandboxd/engine/reader_test.go | 98 +++++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 48 deletions(-) create mode 100644 sandboxd/engine/reader.go create mode 100644 sandboxd/engine/reader_test.go diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 11dd07ed..2c636d6f 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -264,48 +264,12 @@ func (e *Engine) Inspect(ctx context.Context, name string) (rec types.VMRecord, // DialSilkd connects to a VM's silkd through the hybrid-vsock UDS. func (e *Engine) DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, "unix", vsockSocket) - if err != nil { - return nil, err - } - stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) - defer stop() - if _, err = fmt.Fprintf(conn, "CONNECT %d\n", silkdPort); err != nil { - _ = conn.Close() - return nil, fmt.Errorf("write CONNECT: %w", err) - } - r := bufio.NewReaderSize(conn, connectMax) - reply, err := readBufLine(r, connectMax) - if err != nil { - _ = conn.Close() - if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr - } - return nil, fmt.Errorf("read CONNECT reply: %w", err) - } - if !strings.HasPrefix(reply, "OK ") { - _ = conn.Close() - return nil, fmt.Errorf("hybrid vsock CONNECT %d: %s", silkdPort, strings.TrimSpace(reply)) - } - return utils.WithBuffered(conn, r), nil + return e.dialVsock(ctx, vsockSocket, silkdPort) } // Probe polls until silkd completes an info round-trip — the claim-ready signal. func (e *Engine) Probe(ctx context.Context, vsockSocket string, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - var lastErr error - for { - if lastErr = e.infoRoundTrip(ctx, vsockSocket); lastErr == nil { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("silkd probe: %w (last: %v)", ctx.Err(), lastErr) - case <-time.After(probeInterval): - } - } + return e.probe(ctx, vsockSocket, silkdPort, timeout) } // DialGuestPort opens a raw byte stream to 127.0.0.1:port inside the guest. @@ -343,6 +307,50 @@ func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uin return newGuestPortConn(conn, r), nil } +func (e *Engine) dialVsock(ctx context.Context, vsockSocket string, port int) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, "unix", vsockSocket) + if err != nil { + return nil, err + } + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + if _, err = fmt.Fprintf(conn, "CONNECT %d\n", port); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("write CONNECT: %w", err) + } + r := bufio.NewReaderSize(conn, connectMax) + reply, err := readBufLine(r, connectMax) + if err != nil { + _ = conn.Close() + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, fmt.Errorf("read CONNECT reply: %w", err) + } + if !strings.HasPrefix(reply, "OK ") { + _ = conn.Close() + return nil, fmt.Errorf("hybrid vsock CONNECT %d: %s", port, strings.TrimSpace(reply)) + } + return utils.WithBuffered(conn, r), nil +} + +func (e *Engine) probe(ctx context.Context, vsockSocket string, port int, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var lastErr error + for { + if lastErr = e.infoRoundTrip(ctx, vsockSocket, port); lastErr == nil { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("silkd probe: %w (last: %v)", ctx.Err(), lastErr) + case <-time.After(probeInterval): + } + } +} + func (e *Engine) cloneArgs(fromDir, name string, key types.PoolKey) []string { // --pull: a checkpoint/template export carries only COW + memory, not the base image. args := []string{"vm", "clone", "--from-dir", fromDir, argName, name, "--pull", argOutput, formatJSON, e.directIOArg()} @@ -435,8 +443,8 @@ func (e *Engine) runRecord(ctx context.Context, args ...string) (types.VMRecord, return rec, nil } -func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string) error { - conn, err := e.DialSilkd(ctx, vsockSocket) +func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string, port int) error { + conn, err := e.dialVsock(ctx, vsockSocket, port) if err != nil { return err } diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index 31a42986..f2490bb0 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -84,7 +84,7 @@ func TestInfoRoundTripRejectsErrorFrame(t *testing.T) { path := sockPath(t) listenMuxer(t, path, "OK 2048\n", errFrame) - err := New("cocoon", nil, nil, false, false, "").infoRoundTrip(t.Context(), path) + err := New("cocoon", nil, nil, false, false, "").infoRoundTrip(t.Context(), path, silkdPort) if err == nil || !strings.Contains(err.Error(), `info reply type "error"`) { t.Errorf("got %v, want error-frame rejection", err) } diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 09deafe1..c71fabd8 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -96,6 +96,8 @@ type fakeSilkd struct { statMisses map[string]int statErr map[string]string statCalls []string + connects []string + execDetach []bool } func serveFakeSilkd(t *testing.T, path string) *fakeSilkd { @@ -128,10 +130,14 @@ func serveFakeSilkd(t *testing.T, path string) *fakeSilkd { func (f *fakeSilkd) serve(conn net.Conn) { defer func() { _ = conn.Close() }() r := bufio.NewReader(conn) - if _, err := r.ReadString('\n'); err != nil { + connect, err := r.ReadString('\n') + if err != nil { return } - if _, err := io.WriteString(conn, "OK 2048\n"); err != nil { + f.mu.Lock() + f.connects = append(f.connects, strings.TrimSpace(connect)) + f.mu.Unlock() + if _, err = io.WriteString(conn, "OK 2048\n"); err != nil { return } line, err := r.ReadString('\n') @@ -139,16 +145,24 @@ func (f *fakeSilkd) serve(conn net.Conn) { return } var req struct { - Op string `json:"op"` - Path string `json:"path"` - Mode uint32 `json:"mode"` - Argv []string `json:"argv"` - Env map[string]string `json:"env"` + Op string `json:"op"` + Path string `json:"path"` + Mode uint32 `json:"mode"` + Argv []string `json:"argv"` + Env map[string]string `json:"env"` + Detach bool `json:"detach"` } if json.Unmarshal([]byte(line), &req) != nil { return } + if req.Op == "exec" { + f.mu.Lock() + f.execDetach = append(f.execDetach, req.Detach) + f.mu.Unlock() + } switch req.Op { + case "info": + writeFakeSilkdResponse(conn, &wire.InfoResp{Version: "test", Proto: wire.KeepAliveProto}) case "fs_write": f.handleWrite(conn, r, req.Path, req.Mode) case "fs_list": diff --git a/sandboxd/engine/reader.go b/sandboxd/engine/reader.go new file mode 100644 index 00000000..b1aaa8ac --- /dev/null +++ b/sandboxd/engine/reader.go @@ -0,0 +1,84 @@ +package engine + +import ( + "context" + "fmt" + "net" + "slices" + "strings" + "time" + + "github.com/cocoonstack/sandbox/protocol/wire" +) + +const ( + readerPort = 2051 + readerRoot = "/run/reader" + readerView = "/view" + readerSilkd = "/usr/local/bin/silkd" + readerStartTimeout = 30 * time.Second + readerProbeTimeout = 10 * time.Second +) + +// ReaderDisks names the disks attached to a reader VM: the claim's layers, bottom layer first, and its writable copy-on-write disk. +type ReaderDisks struct { + Layers []string + Cow string +} + +// StartReader mounts the attached disks as a read-only overlay of the claim's filesystem and starts a silkd chrooted into it. +func (e *Engine) StartReader(ctx context.Context, vsockSocket string, d ReaderDisks) error { + ctx, cancel := context.WithTimeout(ctx, readerStartTimeout) + defer cancel() + cow := readerRoot + "/cow" + if err := e.MountVolume(ctx, vsockSocket, d.Cow, cow, true); err != nil { + return fmt.Errorf("mount claim disk: %w", err) + } + lower := []string{cow + "/upper"} + for i, layer := range slices.Backward(d.Layers) { + mount := fmt.Sprintf("%s/layer%d", readerRoot, i) + if err := e.MountVolume(ctx, vsockSocket, layer, mount, false); err != nil { + return fmt.Errorf("mount layer %d: %w", i, err) + } + lower = append(lower, mount) + } + if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", readerView); err != nil { + return fmt.Errorf("create view mount point: %w", err) + } + opts := "ro,metacopy=on,redirect_dir=follow,lowerdir=" + strings.Join(lower, ":") + if err := e.silkdExec(ctx, vsockSocket, "mount", "-t", "overlay", "-o", opts, "--", "overlay", readerView); err != nil { + return fmt.Errorf("mount view: %w", err) + } + if err := e.silkdSpawn(ctx, vsockSocket, "env", fmt.Sprintf("SILKD_PORT=%d", readerPort), "chroot", readerView, readerSilkd); err != nil { + return fmt.Errorf("start view silkd: %w", err) + } + return e.probe(ctx, vsockSocket, readerPort, readerProbeTimeout) +} + +// DialReader connects to the silkd StartReader started. +func (e *Engine) DialReader(ctx context.Context, vsockSocket string) (net.Conn, error) { + return e.dialVsock(ctx, vsockSocket, readerPort) +} + +func (e *Engine) silkdSpawn(ctx context.Context, vsockSocket string, argv ...string) error { + s, err := e.dialSilkdSession(ctx, vsockSocket) + if err != nil { + return err + } + defer s.close() + if err = s.send(wire.Exec{Argv: argv, Env: map[string]string{"PATH": guestExecPATH}, Detach: true}); err != nil { + return err + } + frame, err := s.recv() + if err != nil { + return err + } + switch resp := frame.(type) { + case *wire.Started: + return nil + case *wire.ErrorResp: + return fmt.Errorf("silkd %w", resp) + default: + return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) + } +} diff --git a/sandboxd/engine/reader_test.go b/sandboxd/engine/reader_test.go new file mode 100644 index 00000000..c4c652be --- /dev/null +++ b/sandboxd/engine/reader_test.go @@ -0,0 +1,98 @@ +package engine + +import ( + "slices" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/protocol/wire" +) + +func TestStartReaderAssemblesTheViewTopLayerFirst(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureReaderDevices(fake) + e := New("cocoon", nil, nil, false, false, "") + + if err := e.StartReader(t.Context(), sock, ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}); err != nil { + t.Fatalf("StartReader: %v", err) + } + want := [][]string{ + {"mkdir", "-p", "--", "/run/reader/cow"}, + {"mount", "-o", "rw", "--", "/dev/vdb", "/run/reader/cow"}, + {"mkdir", "-p", "--", "/run/reader/layer1"}, + {"mount", "-o", "ro", "--", "/dev/vdd", "/run/reader/layer1"}, + {"mkdir", "-p", "--", "/run/reader/layer0"}, + {"mount", "-o", "ro", "--", "/dev/vdc", "/run/reader/layer0"}, + {"mkdir", "-p", "--", "/view"}, + {"mount", "-t", "overlay", "-o", "ro,metacopy=on,redirect_dir=follow,lowerdir=/run/reader/cow/upper:/run/reader/layer1:/run/reader/layer0", "--", "overlay", "/view"}, + {"env", "SILKD_PORT=2051", "chroot", "/view", "/usr/local/bin/silkd"}, + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != len(want) { + t.Fatalf("guest ran %d commands %v, want %d", len(fake.execCalls), fake.execCalls, len(want)) + } + for i, argv := range want { + if !slices.Equal(fake.execCalls[i], argv) { + t.Errorf("command %d = %v, want %v", i, fake.execCalls[i], argv) + } + } + if last := fake.connects[len(fake.connects)-1]; last != "CONNECT 2051" { + t.Errorf("readiness probe dialed %q, want the view silkd's port", last) + } + for i, detached := range fake.execDetach { + if detached != (i == len(want)-1) { + t.Errorf("command %d detached=%v, want only the view silkd detached", i, detached) + } + } +} + +func TestStartReaderStopsAtTheFirstFailedMount(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureReaderDevices(fake) + fake.mu.Lock() + fake.execFailAt, fake.execCode = 4, 32 + fake.mu.Unlock() + + err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}) + if err == nil || !strings.Contains(err.Error(), "mount layer 1") { + t.Fatalf("StartReader = %v, want the layer 1 mount failure", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != 4 { + t.Errorf("guest ran %d commands after the failed mount, want it to stop at 4: %v", len(fake.execCalls), fake.execCalls) + } +} + +func TestDialReaderConnectsToTheViewPort(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + conn, err := New("cocoon", nil, nil, false, false, "").DialReader(t.Context(), sock) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + _ = conn.Close() + fake.mu.Lock() + defer fake.mu.Unlock() + if !slices.Equal(fake.connects, []string{"CONNECT 2051"}) { + t.Errorf("connects = %v, want one CONNECT 2051", fake.connects) + } +} + +func configureReaderDevices(f *fakeSilkd) { + f.mu.Lock() + defer f.mu.Unlock() + f.block = []wire.DirEntry{ + {Name: "vda", Kind: wire.FileKindSymlink}, + {Name: "vdb", Kind: wire.FileKindSymlink}, + {Name: "vdc", Kind: wire.FileKindSymlink}, + {Name: "vdd", Kind: wire.FileKindSymlink}, + } + for dev, serial := range map[string]string{"vda": "root", "vdb": "rcow", "vdc": "rl0", "vdd": "rl1"} { + f.serial["/sys/block/"+dev+"/serial"] = []byte(serial + "\n") + f.stat["/dev/"+dev] = true + } +} From 2a39c53b55cedf839ac5370ebd0e5cc9dea4ecb8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:49:11 +0800 Subject: [PATCH 05/28] config: reader block names the pool that serves sleeping claims' file reads --- sandboxd/config/config.go | 27 ++++++++++++++++++++++++++- sandboxd/config/config_test.go | 19 +++++++++++++++++++ sandboxd/config/reload_test.go | 3 ++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index bcc4d57e..c678a588 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -164,6 +164,16 @@ type VolumeSpec struct { Tenants []string `json:"tenants,omitempty"` } +// ReaderSpec names the none-lane pool whose warm VMs serve read-only file calls on a hibernated or archived claim. +type ReaderSpec struct { + Template string `json:"template"` + Size types.Size `json:"size,omitempty"` +} + +func (r ReaderSpec) Key() types.PoolKey { + return types.PoolKey{Template: r.Template, Size: r.Size}.Defaulted() +} + // DiskReserve refuses claims, wakes and refills while a listed directory's filesystem has less than Free left. type DiskReserve struct { Free string `json:"free"` @@ -268,6 +278,7 @@ type Config struct { AuditLog bool `json:"audit_log,omitzero"` DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` + Reader *ReaderSpec `json:"reader,omitempty"` Volumes []VolumeSpec `json:"volumes,omitempty"` Mesh *MeshConfig `json:"mesh,omitempty"` @@ -366,7 +377,7 @@ func (c *Config) validate() error { if c.CheckpointPeerHeal && c.APIToken == "" { return fmt.Errorf("checkpoint_peer_heal requires api_token: without it resolveScope leaves the raw checkpoint blob GET reachable with no credential") } - for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { + for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateReader, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { if err := check(); err != nil { return err } @@ -613,6 +624,20 @@ func (c *Config) validateDiskReserve() error { return nil } +func (c *Config) validateReader() error { + if c.Reader == nil { + return nil + } + key := c.Reader.Key() + if err := key.Validate(); err != nil { + return fmt.Errorf("reader: %w", err) + } + if !slices.ContainsFunc(c.Pools, func(p PoolSpec) bool { return p.PoolKey == key && p.Warm > 0 }) { + return fmt.Errorf("reader %s %s/%s must be a configured pool with warm of at least 1", key.Template, key.Net, key.Size) + } + return nil +} + func (c *Config) validateMetaStore() error { ms := c.MetaStore switch { diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 16b385a7..36540738 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -167,6 +167,11 @@ func TestLoadRejectsInvalid(t *testing.T) { {"duplicate volume name", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img"},{"name":"data","path":"/srv/datasets/b.img"}]}`, "duplicate volume"}, {"relative volume path", `{"pools":[],"volumes":[{"name":"data","path":"datasets/a.img"}]}`, "path must be absolute"}, {"bad volume directio", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img","directio":"yes"}]}`, "directio must be"}, + {"reader without a pool", `{"pools":[{"template":"rt:24.04"}],"reader":{"template":"reader:24.04"}}`, "must be a configured pool"}, + {"reader on another size", `{"pools":[{"template":"rt:24.04"}],"reader":{"template":"rt:24.04","size":"large"}}`, "must be a configured pool"}, + {"reader pool without warm", `{"pools":[{"template":"rt:24.04","warm":0}],"reader":{"template":"rt:24.04"}}`, "warm of at least 1"}, + {"reader without a template", `{"pools":[],"reader":{}}`, "reader: template"}, + {"reader typo", `{"pools":[{"template":"rt:24.04"}],"reader":{"template":"rt:24.04","sizes":"small"}}`, "unknown object member"}, } { t.Run(tt.name, func(t *testing.T) { _, err := Load(writeConfig(t, tt.body)) @@ -280,6 +285,20 @@ func TestLoadAcceptsPreviewAdvertiseShapes(t *testing.T) { } } +func TestLoadAcceptsReaderOnAConfiguredPool(t *testing.T) { + cfg, err := Load(writeConfig(t, `{"pools":[{"template":"reader:24.04","warm":2}],"reader":{"template":"reader:24.04"}}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + want := types.PoolKey{Template: "reader:24.04", Net: types.NetNone, Size: types.SizeSmall} + if got := cfg.Reader.Key(); got != want { + t.Errorf("reader key = %+v, want the defaulted pool key %+v", got, want) + } + if cfg, err = Load(writeConfig(t, `{"pools":[]}`)); err != nil || cfg.Reader != nil { + t.Errorf("a config without reader loaded as %+v, %v, want the feature off", cfg.Reader, err) + } +} + func TestLoadAcceptsVolumes(t *testing.T) { path := writeConfig(t, `{"pools":[],"volumes":[ {"name":"imagenet","path":"/srv/datasets/imagenet.img"}, diff --git a/sandboxd/config/reload_test.go b/sandboxd/config/reload_test.go index e7103a80..905b7370 100644 --- a/sandboxd/config/reload_test.go +++ b/sandboxd/config/reload_test.go @@ -56,7 +56,8 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { restart := *cur restart.Listen, restart.Bridges, restart.EgressUsageBytes = "127.0.0.1:8888", []string{"br1"}, true - if _, _, err := cur.ReloadDiff(&restart); err == nil || err.Error() != "listen, bridges change only at a restart" { + restart.Reader = &ReaderSpec{Template: "desktop:v12"} + if _, _, err := cur.ReloadDiff(&restart); err == nil || err.Error() != "listen, bridges, reader change only at a restart" { t.Errorf("ReloadDiff over restart fields: %v", err) } if changed, ignored, err := cur.ReloadDiff(cur); err != nil || changed != nil || ignored != nil { From 6415e4e9b17312b754eb1e2c962edfe49271b1be Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:55:46 +0800 Subject: [PATCH 06/28] pool: serve a sleeping claim's files from a reader VM A reader is a warm VM that gets the claim's layer blobs read-only and a private reflinked copy of its disk, mounts them as a read-only overlay and runs a silkd chrooted into it. Setup is shared per claim, the reader is kept for 30s after its last connection, and a hibernate syncs the guest first so the snapshot disk holds the page cache. --- e2e/fakeengine_test.go | 37 ++- sandboxd/pool/hibernate.go | 39 ++- sandboxd/pool/pool.go | 10 + sandboxd/pool/pool_test.go | 34 +++ sandboxd/pool/reader.go | 324 +++++++++++++++++++++ sandboxd/pool/reader_test.go | 546 +++++++++++++++++++++++++++++++++++ sandboxd/pool/reconcile.go | 1 + 7 files changed, 986 insertions(+), 5 deletions(-) create mode 100644 sandboxd/pool/reader.go create mode 100644 sandboxd/pool/reader_test.go diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index afef49d4..e4366008 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "sync" "time" @@ -28,6 +29,9 @@ type fakeEngine struct { volumeVMs map[string]bool guestEnvs map[string]string seq int + + readerRoot string + readerStarts []string } func newFakeEngine(dir string) *fakeEngine { @@ -85,7 +89,38 @@ func (f *fakeEngine) ReconcileStaleCreate(context.Context, string) (engine.Stale func (f *fakeEngine) SnapshotSave(_ context.Context, _, _ string) error { return nil } func (f *fakeEngine) SnapshotExport(_ context.Context, _, toDir string) error { - return os.MkdirAll(toDir, 0o750) + if err := os.MkdirAll(toDir, 0o750); err != nil { + return err + } + layer := filepath.Join(f.dir, "layer.erofs") + sidecar := `{"storage_configs":[{"path":` + strconv.Quote(layer) + `,"role":"layer"}]}` + files := map[string]string{ + layer: "erofs", + filepath.Join(toDir, "cow.raw"): "disk", + filepath.Join(toDir, "cocoon.json"): sidecar, + } + for path, body := range files { + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + return err + } + } + return nil +} + +func (f *fakeEngine) StartReader(_ context.Context, vsockSocket string, _ engine.ReaderDisks) error { + f.mu.Lock() + defer f.mu.Unlock() + f.readerStarts = append(f.readerStarts, vsockSocket) + return nil +} + +func (f *fakeEngine) DialReader(_ context.Context, _ string) (net.Conn, error) { + f.mu.Lock() + root := f.readerRoot + f.mu.Unlock() + client, server := net.Pipe() + go silkdtest.NewFake(root).ServeConn(server) + return client, nil } func (f *fakeEngine) SnapshotRemove(_ context.Context, _ string) error { return nil } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 478b6aa2..be4d5a30 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -26,6 +26,15 @@ func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { // WakeAgentSocket resolves a data-plane connection's guest socket; the returned func ends the hold it takes, and the channel closes when the stream must end. func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) { + return m.agentSocket(ctx, id, token, false) +} + +// WakeAgentSocketOrRead is WakeAgentSocket, except that with a reader a sleeping claim answers ErrSleeping instead of waking. +func (m *Manager) WakeAgentSocketOrRead(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) { + return m.agentSocket(ctx, id, token, m.readers != nil) +} + +func (m *Manager) agentSocket(ctx context.Context, id, token string, readerOK bool) (string, <-chan struct{}, func(), error) { sb, ok := m.claim(id, token) if !ok { return "", nil, nil, ErrUnknownSandbox @@ -34,13 +43,22 @@ func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string if err != nil { return "", nil, nil, err } - sb.Touch() + if !readerOK { + sb.Touch() + } sb.Hold() - sock, err := m.wakeResolved(ctx, sb) - if err != nil { + sock, err := m.wakeOrRead(ctx, sb, readerOK) + switch { + case errors.Is(err, ErrSleeping): + sb.UnholdIdle() + return "", nil, nil, err + case err != nil: sb.Unhold() return "", nil, nil, err } + if readerOK { + sb.Touch() + } return sock, fenced, sb.Unhold, nil } @@ -87,6 +105,9 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error m.mu.Unlock() return fmt.Errorf("hibernate %s: persist intent: %w", sb.ID, err) } + if m.readers != nil { + m.syncForReader(ctx, sb) + } if hibErr := m.eng.Hibernate(ctx, sb.VMName, snap); hibErr != nil { // the engine can report failure after the snapshot landed adopted, resolveErr := m.settlePendingSnap(ctx, sb) @@ -120,9 +141,16 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error // wakeResolved is the wake body shared by the token and id-only entry points. func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, error) { + return m.wakeOrRead(ctx, sb, false) +} + +func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bool) (string, error) { sb.Transition.Lock() defer sb.Transition.Unlock() if sb.ArchiveCk != "" { + if readerOK { + return "", ErrSleeping + } return m.wakeArchived(ctx, sb) } if sb.Failed != "" { @@ -143,6 +171,9 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, if sb.Key.Net == types.NetEgress { return "", fmt.Errorf("wake %s: egress lane cannot resume from hibernation", sb.ID) } + if readerOK { + return "", ErrSleeping + } if err := m.diskErr(); err != nil { return "", fmt.Errorf("wake %s: %w", sb.ID, err) } @@ -174,7 +205,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, m.counters.wakeNanos.Add(uint64(time.Since(wakeStart))) //nolint:gosec // durations are positive m.recordUsage(ctx, usageEvent{Event: "wake", ID: sb.ID, VMName: sb.VMName}) if proxyErr := m.out.ArmProxy(ctx, sb); proxyErr != nil { - log.WithFunc("pool.wakeResolved").Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) + log.WithFunc("pool.wakeOrRead").Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) } if m.disarmIfReleased(sb) { if err == nil { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 5c64479f..4de4ae88 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -110,7 +110,10 @@ var ( ErrFailed = errors.New("sandbox failed: its VMM is down; release it, or wake it to cold-boot its disk") ErrNoInstanceMetadata = engine.ErrNoInstanceMetadata ErrQuota = errors.New("node claim quota reached") + ErrSleeping = errors.New("sandbox is hibernated or archived and a reader may serve it") + ErrNoReader = errors.New("no reader for the sandbox") + errNotSleeping = errors.New("sandbox is no longer hibernated or archived") errWokeMeanwhile = errors.New("woke between sweep and hibernate") errNoEgressTap = errors.New("egress-lane claim has no lockable tap") ) @@ -147,6 +150,8 @@ type Engine interface { MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error SyncGuest(ctx context.Context, vsockSocket string) error + StartReader(ctx context.Context, vsockSocket string, d engine.ReaderDisks) error + DialReader(ctx context.Context, vsockSocket string) (net.Conn, error) WriteInstanceMetadata(ctx context.Context, vsockSocket string, doc []byte) error WriteGuestEnv(ctx context.Context, vsockSocket string, doc []byte) error } @@ -399,6 +404,8 @@ type Manager struct { out *outbound.Host + readers *readerSet + mu sync.Mutex pools map[types.PoolKey]*pool claimed map[string]*types.Sandbox @@ -457,6 +464,9 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg refillKick: make(chan struct{}, 1), healSem: make(chan struct{}, maxConcurrentHeals), } + if cfg.Reader != nil { + m.readers = &readerSet{key: cfg.Reader.Key(), live: map[string]*reader{}} + } for _, volume := range cfg.Volumes { m.volumes[volume.Name] = catalogVolume{ disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 81b2269e..e44ca5b2 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1220,6 +1220,11 @@ type fakeEngine struct { hibernates, restores, snapRemoves []string snapSaves, exports, snapshots []string exportContent []byte + exportFiles map[string]string + startedReaders []engine.ReaderDisks + readerDials []string + startReaderErr error + readerDialErr error caInstalls []string warmups [][]string warmupSocks []string @@ -1393,6 +1398,14 @@ func (f *fakeEngine) SnapshotExport(_ context.Context, snapName, toDir string) e if err := os.MkdirAll(toDir, 0o750); err != nil { return err } + f.mu.Lock() + files := maps.Clone(f.exportFiles) + f.mu.Unlock() + for name, body := range files { + if err := os.WriteFile(filepath.Join(toDir, name), []byte(body), 0o600); err != nil { + return err + } + } if content == nil { return nil } @@ -1673,6 +1686,27 @@ func (f *fakeEngine) SyncGuest(_ context.Context, vsockSocket string) error { return nil } +func (f *fakeEngine) StartReader(_ context.Context, vsockSocket string, d engine.ReaderDisks) error { + f.mu.Lock() + defer f.mu.Unlock() + f.startedReaders = append(f.startedReaders, d) + f.volumeOps = append(f.volumeOps, "reader-start:"+vsockSocket) + return f.startReaderErr +} + +func (f *fakeEngine) DialReader(_ context.Context, vsockSocket string) (net.Conn, error) { + f.mu.Lock() + f.readerDials = append(f.readerDials, vsockSocket) + err := f.readerDialErr + f.mu.Unlock() + if err != nil { + return nil, err + } + conn, peer := net.Pipe() + _ = peer.Close() + return conn, nil +} + func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go new file mode 100644 index 00000000..ac7580c1 --- /dev/null +++ b/sandboxd/pool/reader.go @@ -0,0 +1,324 @@ +package pool + +import ( + "cmp" + "context" + "encoding/json/v2" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sync" + "time" + + "github.com/projecteru2/core/log" + "golang.org/x/sync/singleflight" + + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +const ( + readerIdle = 30 * time.Second + readerSetupTimeout = 5 * time.Minute + readerDir = "readers" + readerCowDisk = "rcow" + readerCowFile = "cow.raw" + readerSidecarFile = "cocoon.json" + layerRole = "layer" +) + +type readerSet struct { + key types.PoolKey + mu sync.Mutex + live map[string]*reader + flights singleflight.Group +} + +type reader struct { + claim string + source string + vm string + sock string + dir string + conns int + idle *time.Timer +} + +// DialReader opens a silkd connection that serves a sleeping claim's files read-only from a reader VM, never waking the claim; source names the sleep it serves. +func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Conn, source string, err error) { + sb, ok := m.claim(id, token) + if !ok { + return nil, "", ErrUnknownSandbox + } + if m.readers == nil { + return nil, "", ErrNoReader + } + r, err := m.acquireReader(ctx, sb) + if err != nil { + return nil, "", fmt.Errorf("%w: %w", ErrNoReader, err) + } + conn, err = m.eng.DialReader(ctx, r.sock) + if err != nil { + m.releaseReader(r) + m.retireReader(r) + return nil, "", fmt.Errorf("%w: %w", ErrNoReader, err) + } + return &heldConn{Conn: conn, release: sync.OnceFunc(func() { m.releaseReader(r) })}, r.source, nil +} + +// SleepingSource names the hibernate snapshot or archive record a reader would serve, or "" when the claim is awake, failed or mid-transition. +func (m *Manager) SleepingSource(id, token string) string { + m.mu.Lock() + defer m.mu.Unlock() + sb, ok := m.authed(id, token) + if !ok { + return "" + } + return sleepSource(sb) +} + +func (m *Manager) acquireReader(ctx context.Context, sb *types.Sandbox) (*reader, error) { + rs := m.readers + for { + m.mu.Lock() + want := sleepSource(sb) + m.mu.Unlock() + if want == "" { + return nil, errNotSleeping + } + rs.mu.Lock() + r := rs.live[sb.ID] + if r != nil && r.source == want { + r.conns++ + if r.idle != nil { + r.idle.Stop() + r.idle = nil + } + rs.mu.Unlock() + return r, nil + } + rs.mu.Unlock() + if r != nil { + m.retireReader(r) + } + _, err, _ := rs.flights.Do(sb.ID, func() (any, error) { + setupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), readerSetupTimeout) + defer cancel() + return nil, m.openReader(setupCtx, sb) + }) + if err != nil { + return nil, err + } + } +} + +func (m *Manager) releaseReader(r *reader) { + rs := m.readers + rs.mu.Lock() + defer rs.mu.Unlock() + r.conns-- + if r.conns == 0 && rs.live[r.claim] == r { + r.idle = time.AfterFunc(readerIdle, func() { m.reapReader(r) }) + } +} + +func (m *Manager) reapReader(r *reader) { + if m.unregisterReader(r, false) { + m.destroyReader(context.Background(), r) + } +} + +func (m *Manager) retireReader(r *reader) { + if m.unregisterReader(r, true) { + m.destroyReader(context.Background(), r) + } +} + +func (m *Manager) unregisterReader(r *reader, force bool) bool { + rs := m.readers + rs.mu.Lock() + defer rs.mu.Unlock() + if rs.live[r.claim] != r || (!force && r.conns > 0) { + return false + } + delete(rs.live, r.claim) + return true +} + +func (m *Manager) openReader(ctx context.Context, sb *types.Sandbox) error { + start := time.Now() + r := &reader{claim: sb.ID, dir: filepath.Join(m.dataDir, readerDir, sb.ID+"-"+randHex(3))} + layers, err := m.readerSource(ctx, sb, r) + if err != nil { + _ = os.RemoveAll(r.dir) + return err + } + vm, err := m.takeReaderVM() + if err != nil { + _ = os.RemoveAll(r.dir) + return err + } + r.vm, r.sock = vm.VMName, vm.VsockSocket + if err = m.startReader(ctx, r, layers); err != nil { + m.destroyReader(ctx, r) + return err + } + rs := m.readers + rs.mu.Lock() + rs.live[sb.ID] = r + rs.mu.Unlock() + log.WithFunc("pool.openReader").Infof(ctx, "reader %s for sleeping claim %s ready in %s", r.vm, sb.ID, time.Since(start).Round(time.Millisecond)) + return nil +} + +func (m *Manager) readerSource(ctx context.Context, sb *types.Sandbox, r *reader) ([]string, error) { + archived, err := m.captureSleeping(ctx, sb, r) + if err != nil { + return nil, err + } + sidecar := filepath.Join(r.dir, readerSidecarFile) + if archived { + if sidecar, err = m.fetchReaderDisk(ctx, r); err != nil { + return nil, err + } + } + return readerLayers(sidecar) +} + +// captureSleeping copies a hibernated snapshot under the Transition lock, which the wake that consumes the snapshot takes. +func (m *Manager) captureSleeping(ctx context.Context, sb *types.Sandbox, r *reader) (archived bool, err error) { + sb.Transition.Lock() + defer sb.Transition.Unlock() + if sb.HibernateSnap == "" && sb.ArchiveCk == "" { + return false, errNotSleeping + } + if sb.HibernateSnap == "" { + if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { + return false, err + } + } + if err = m.diskErr(); err != nil { + return false, err + } + if sb.HibernateSnap == "" { + r.source = sb.ArchiveCk + return true, nil + } + r.source = sb.HibernateSnap + if err = m.eng.SnapshotExport(ctx, r.source, r.dir); err != nil { + return false, fmt.Errorf("export snapshot: %w", err) + } + return false, utils.RemoveDirEntries(r.dir, func(name string) bool { return name != readerCowFile && name != readerSidecarFile }) +} + +func (m *Manager) fetchReaderDisk(ctx context.Context, r *reader) (string, error) { + disk, err := m.ckpts.FetchFile(ctx, r.source, readerCowFile) + if err != nil { + return "", archiveFetchErr(err) + } + sidecar, err := m.ckpts.FetchFile(ctx, r.source, readerSidecarFile) + if err != nil { + return "", archiveFetchErr(err) + } + if err = os.MkdirAll(r.dir, 0o750); err != nil { + return "", err + } + if err = utils.CloneFile(disk, filepath.Join(r.dir, readerCowFile)); err != nil { + return "", fmt.Errorf("copy archive disk: %w", err) + } + return sidecar, nil +} + +func (m *Manager) takeReaderVM() (*types.Sandbox, error) { + m.mu.Lock() + var sb *types.Sandbox + if p, ok := m.activePool(m.readers.key); ok { + if n := len(p.warm); n > 0 { + sb = p.warm[n-1] + p.warm = p.warm[:n-1] + } + } + m.mu.Unlock() + m.kickRefill() + if sb == nil { + return nil, ErrNoWarm + } + return sb, nil +} + +func (m *Manager) startReader(ctx context.Context, r *reader, layers []string) error { + disks := engine.ReaderDisks{Cow: readerCowDisk} + for i, blob := range layers { + name := fmt.Sprintf("rl%d", i) + if err := m.eng.DiskAttach(ctx, r.vm, engine.VolumeSpec{Name: name, Path: blob}); err != nil { + return fmt.Errorf("attach layer %d: %w", i, err) + } + disks.Layers = append(disks.Layers, name) + } + if err := m.eng.DiskAttach(ctx, r.vm, engine.VolumeSpec{Name: readerCowDisk, Path: filepath.Join(r.dir, readerCowFile), RW: true}); err != nil { + return fmt.Errorf("attach claim disk: %w", err) + } + return m.eng.StartReader(ctx, r.sock, disks) +} + +func (m *Manager) destroyReader(ctx context.Context, r *reader) { + m.destroy(ctx, r.vm) + if err := os.RemoveAll(r.dir); err != nil { + log.WithFunc("pool.destroyReader").Warnf(ctx, "remove reader dir %s: %v", r.dir, err) + } +} + +func (m *Manager) syncForReader(ctx context.Context, sb *types.Sandbox) { + if err := m.eng.SyncGuest(ctx, sb.VsockSocket); err != nil { + log.WithFunc("pool.syncForReader").Warnf(ctx, "sync %s before hibernate: %v", sb.ID, err) + } +} + +// sleepSource names a settled sleep's snapshot or record; callers hold m.mu. +func sleepSource(sb *types.Sandbox) string { + if sb.Failed != "" || sb.PendingSnap != "" { + return "" + } + return cmp.Or(sb.HibernateSnap, sb.ArchiveCk) +} + +func archiveFetchErr(err error) error { + if errors.Is(err, store.ErrNotFound) { + return errNotSleeping + } + return fmt.Errorf("fetch archive: %w", err) +} + +func readerLayers(sidecar string) ([]string, error) { + raw, err := os.ReadFile(sidecar) //nolint:gosec // path from our own export or fetch + if err != nil { + return nil, fmt.Errorf("read snapshot sidecar: %w", err) + } + var meta struct { + StorageConfigs []struct { + Path string `json:"path"` + Role string `json:"role"` + } `json:"storage_configs"` + } + if err = json.Unmarshal(raw, &meta); err != nil { + return nil, fmt.Errorf("parse snapshot sidecar: %w", err) + } + var layers []string + for _, sc := range meta.StorageConfigs { + if sc.Role != layerRole { + continue + } + if _, err = os.Stat(sc.Path); err != nil { + return nil, fmt.Errorf("layer blob: %w", err) + } + layers = append(layers, sc.Path) + } + if len(layers) == 0 { + return nil, errors.New("snapshot sidecar names no layers") + } + return layers, nil +} diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go new file mode 100644 index 00000000..5ec17b6f --- /dev/null +++ b/sandboxd/pool/reader_test.go @@ -0,0 +1,546 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +var readerKey = types.PoolKey{Template: "reader:24.04", Net: types.NetNone, Size: types.SizeSmall} + +func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { + eng := newFakeEngine() + layers := readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + if _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + t.Fatalf("WakeAgentSocketOrRead = %v, want ErrSleeping", err) + } + if sb.Busy() { + t.Error("a deferred wake left the claim held") + } + conn, source, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + defer func() { _ = conn.Close() }() + + if source != sb.HibernateSnap || source == "" { + t.Errorf("reader source %q, want the claim's hibernate snapshot %q", source, sb.HibernateSnap) + } + if len(eng.restores) != 0 || sb.HibernateSnap == "" { + t.Errorf("claim woke: restores=%v hibernate snapshot=%q", eng.restores, sb.HibernateSnap) + } + if got, want := readerOps(eng), []string{"attach:rl0", "attach:rl1", "attach:rcow", "reader-start:/vsock/sbx-reader-1"}; !slices.Equal(got, want) { + t.Errorf("reader setup ran %v, want %v", got, want) + } + cow := eng.volumeSpecs[len(eng.volumeSpecs)-1] + wantSpecs := []engine.VolumeSpec{{Name: "rl0", Path: layers[0]}, {Name: "rl1", Path: layers[1]}, {Name: "rcow", Path: cow.Path, RW: true}} + if !slices.Equal(eng.volumeSpecs, wantSpecs) { + t.Errorf("attached %+v, want read-only layers then the writable copy %+v", eng.volumeSpecs, wantSpecs) + } + if want := (engine.ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}); len(eng.startedReaders) != 1 || !slices.Equal(eng.startedReaders[0].Layers, want.Layers) || eng.startedReaders[0].Cow != want.Cow { + t.Errorf("view started with %+v, want %+v", eng.startedReaders, want) + } + if got, readErr := os.ReadFile(cow.Path); readErr != nil || string(got) != "disk-bytes" { + t.Errorf("reader disk %q, %v, want the snapshot's copy-on-write bytes", got, readErr) + } + if entries, _ := os.ReadDir(filepath.Dir(cow.Path)); len(entries) != 2 { + t.Errorf("reader dir holds %d entries, want only the disk and its sidecar", len(entries)) + } + if n := len(m.pools[readerKey].warm); n != 0 { + t.Errorf("reader pool holds %d warm VMs, want the one reader taken", n) + } +} + +func TestReaderCallsShareOneSetup(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + eng.exportStall = make(chan struct{}) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1", "sbx-reader-2") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + const callers = 8 + var wg sync.WaitGroup + for range callers { + wg.Go(func() { + conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Errorf("DialReader: %v", err) + return + } + _ = conn.Close() + }) + } + waitFor(t, func() bool { eng.mu.Lock(); defer eng.mu.Unlock(); return len(eng.exports) > 0 }) + close(eng.exportStall) + wg.Wait() + + eng.mu.Lock() + defer eng.mu.Unlock() + if len(eng.exports) != 1 || len(eng.startedReaders) != 1 { + t.Errorf("%d callers ran %d exports and %d reader setups, want one of each", callers, len(eng.exports), len(eng.startedReaders)) + } + if len(eng.readerDials) != callers { + t.Errorf("%d dials, want one per caller", len(eng.readerDials)) + } +} + +func TestReaderIsReleasedAfterItsIdleWindow(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + dialAndClose := func() { + t.Helper() + conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + _ = conn.Close() + } + dialAndClose() + dir := filepath.Dir(eng.volumeSpecs[len(eng.volumeSpecs)-1].Path) + + time.Sleep(readerIdle * 2 / 3) + dialAndClose() + time.Sleep(readerIdle * 2 / 3) + synctest.Wait() + if removes, setups := readerLifecycle(eng); slices.Contains(removes, "sbx-reader-1") || setups != 1 { + t.Fatalf("reader released inside its window after a reuse: removes=%v setups=%d", removes, setups) + } + + time.Sleep(readerIdle) + synctest.Wait() + if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { + t.Errorf("removes=%v, want the idle reader's VM destroyed", removes) + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Errorf("reader dir survived the release: %v", err) + } + m.readers.mu.Lock() + defer m.readers.mu.Unlock() + if len(m.readers.live) != 0 { + t.Error("released reader is still registered") + } + }) +} + +func TestReaderKeepsTheClaimsLatestSleepOnly(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + first, firstSource, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + _ = first.Close() + if _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("wake: %v", err) + } + if got := m.SleepingSource(sb.ID, sb.Token); got != "" { + t.Fatalf("SleepingSource of a running claim = %q, want none", got) + } + if _, _, err = m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Fatalf("DialReader of a running claim = %v, want ErrNoReader", err) + } + hibernateClaim(t, m, sb) + + second, secondSource, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader after the second sleep: %v", err) + } + _ = second.Close() + if secondSource == firstSource { + t.Errorf("both sleeps served from %q", firstSource) + } + if len(eng.exports) != 2 || len(eng.startedReaders) != 2 { + t.Errorf("exports=%d setups=%d, want a fresh reader per sleep", len(eng.exports), len(eng.startedReaders)) + } + if !slices.Contains(eng.removes, "sbx-reader-1") { + t.Errorf("removes=%v, want the first sleep's reader destroyed", eng.removes) + } +} + +func TestReaderServesAnArchivedClaimFromTwoFilesOfItsRecord(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + counting := &fetchCountingStore{Store: m.ckpts} + m.ckpts = counting + sb := mustClaim(t, m, testKey) + mustArchive(t, m, sb) + clones := len(eng.clones) + len(eng.colds) + + if _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + t.Fatalf("WakeAgentSocketOrRead of an archived claim = %v, want ErrSleeping", err) + } + conn, source, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + defer func() { _ = conn.Close() }() + + m.mu.Lock() + ck := sb.ArchiveCk + m.mu.Unlock() + if ck == "" || source != ck { + t.Errorf("reader source %q, want the archive record %q", source, ck) + } + if got, want := counting.files(), []string{"cocoon.json", "cow.raw"}; !slices.Equal(got, want) { + t.Errorf("fetched files %v, want only %v", got, want) + } + if counting.fetches != 0 { + t.Errorf("%d whole-record fetches, want none", counting.fetches) + } + if n := len(eng.clones) + len(eng.colds); n != clones || len(eng.restores) != 0 { + t.Errorf("an archived read booted a VM: clones/colds %d->%d, restores %v", clones, n, eng.restores) + } + if !ckExists(t, m, ck) { + t.Error("the archive record was consumed") + } + cow := eng.volumeSpecs[len(eng.volumeSpecs)-1] + if got, readErr := os.ReadFile(cow.Path); readErr != nil || string(got) != "disk-bytes" { + t.Errorf("reader disk %q, %v, want the record's bytes", got, readErr) + } + if !strings.HasPrefix(cow.Path, filepath.Join(m.dataDir, readerDir)) { + t.Errorf("reader attached %s, want a private copy under %s", cow.Path, readerDir) + } +} + +func TestReaderRefusesWhatAWakeRefuses(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + cfg := &config.Config{ + DataDir: t.TempDir(), APIToken: "root", Reader: &config.ReaderSpec{Template: readerKey.Template}, + Pools: []config.PoolSpec{archivePool(0), {PoolKey: readerKey, Warm: 1}}, + } + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatalf("setup manager: %v", err) + } + seedReaderVMs(m, "sbx-reader-1") + putTenants(t, m, tenants.TenantSpec{Name: "acme", Token: "acme-tok"}) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "acme"}) + if err != nil { + t.Fatalf("claim: %v", err) + } + mustArchive(t, m, sb) + if err = m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatalf("DeleteTenant: %v", err) + } + + if _, _, err = m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) || !errors.Is(err, ErrTenantRemoved) { + t.Errorf("DialReader of a removed tenant's archive = %v, want ErrNoReader wrapping ErrTenantRemoved", err) + } + if len(eng.startedReaders) != 0 || len(m.pools[readerKey].warm) != 1 { + t.Errorf("a refused read still took a reader: setups=%d warm=%d", len(eng.startedReaders), len(m.pools[readerKey].warm)) + } +} + +func TestReaderRefusesWhileTheDiskIsLow(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + m.diskLow.Store(true) + + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) || !errors.Is(err, ErrQuota) { + t.Errorf("DialReader on a full disk = %v, want ErrNoReader wrapping ErrQuota", err) + } + if len(eng.exports) != 0 || len(m.pools[readerKey].warm) != 1 { + t.Errorf("a refused read still exported (%d) or took a reader (%d warm)", len(eng.exports), len(m.pools[readerKey].warm)) + } +} + +func TestReaderSetupFailureFallsBackAndCleansUp(t *testing.T) { + for _, tt := range []struct { + name string + setup func(*fakeEngine) + warm []string + }{ + {"attach fails", func(e *fakeEngine) { e.diskAttachErrFor = "rcow" }, []string{"sbx-reader-1"}}, + {"view fails", func(e *fakeEngine) { e.startReaderErr = errors.New("mount failed") }, []string{"sbx-reader-1"}}, + {"no warm reader", func(*fakeEngine) {}, nil}, + } { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + tt.setup(eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, tt.warm...) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Fatalf("DialReader = %v, want ErrNoReader", err) + } + if len(tt.warm) > 0 && !slices.Contains(eng.removes, "sbx-reader-1") { + t.Errorf("removes=%v, want the failed reader's VM destroyed", eng.removes) + } + if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { + t.Errorf("a failed setup left %d entries in the reader dir", len(entries)) + } + if len(m.readers.live) != 0 || sb.HibernateSnap == "" || len(eng.restores) != 0 { + t.Errorf("failed setup changed state: live=%d snap=%q restores=%v", len(m.readers.live), sb.HibernateSnap, eng.restores) + } + }) + } +} + +func TestSleepingSourceNamesOnlyASettledSleepOfTheCallersClaim(t *testing.T) { + eng := newFakeEngine() + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + if got := m.SleepingSource(sb.ID, sb.Token); got != "" { + t.Errorf("a running claim sleeps from %q, want none", got) + } + hibernateClaim(t, m, sb) + snap := sb.HibernateSnap + if got := m.SleepingSource(sb.ID, sb.Token); got != snap || snap == "" { + t.Errorf("a hibernated claim sleeps from %q, want its snapshot %q", got, snap) + } + if got := m.SleepingSource(sb.ID, "wrong"); got != "" { + t.Errorf("a wrong token read the sleep: %q", got) + } + for name, mutate := range map[string]func(){ + "transition in flight": func() { sb.PendingSnap = "sbx-hib-next" }, + "failed": func() { sb.Failed = "vmm down" }, + } { + m.mu.Lock() + mutate() + m.mu.Unlock() + if got := m.SleepingSource(sb.ID, sb.Token); got != "" { + t.Errorf("%s: sleeps from %q, want none", name, got) + } + m.mu.Lock() + sb.PendingSnap, sb.Failed = "", "" + m.mu.Unlock() + } +} + +func TestReaderThatStopsAnsweringIsRetired(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + eng.readerDialErr = errors.New("vsock refused") + + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Fatalf("DialReader of a dead reader = %v, want ErrNoReader", err) + } + if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { + t.Errorf("removes=%v, want the dead reader's VM destroyed", removes) + } + eng.readerDialErr = nil + if conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); err != nil { + t.Errorf("DialReader after the retirement = %v, want a fresh reader", err) + } else { + _ = conn.Close() + } + if _, setups := readerLifecycle(eng); setups != 2 { + t.Errorf("%d reader setups, want the retired reader replaced", setups) + } +} + +func TestReaderOffWakesAsBefore(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Errorf("DialReader with the reader off = %v, want ErrNoReader", err) + } + sock, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + if err != nil || sock == "" { + t.Fatalf("WakeAgentSocketOrRead with the reader off = %q, %v, want a wake", sock, err) + } + defer done() + if len(eng.restores) != 1 || sb.HibernateSnap != "" { + t.Errorf("restores=%v snapshot=%q, want the claim woken", eng.restores, sb.HibernateSnap) + } +} + +func TestReaderOnLeavesRunningClaimsAlone(t *testing.T) { + eng := newFakeEngine() + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + + sock, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + if err != nil || sock != sb.VsockSocket { + t.Fatalf("WakeAgentSocketOrRead of a running claim = %q, %v, want its own socket", sock, err) + } + defer done() + if !sb.Busy() { + t.Error("the connection did not hold the running claim") + } + if len(eng.restores) != 0 || len(eng.startedReaders) != 0 { + t.Errorf("a running claim triggered restores=%v setups=%d", eng.restores, len(eng.startedReaders)) + } +} + +func TestHibernateSyncsTheGuestOnlyWithTheReaderOn(t *testing.T) { + for _, on := range []bool{true, false} { + t.Run(fmt.Sprintf("reader=%t", on), func(t *testing.T) { + eng := newFakeEngine() + var m *Manager + if on { + m = newReaderManager(t, eng) + } else { + m = newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + } + sb := mustClaim(t, m, testKey) + eng.hibernateStall = make(chan struct{}) + hibernated := make(chan struct{}) + go func() { + hibernateClaim(t, m, sb) + close(hibernated) + }() + waitFor(t, func() bool { eng.mu.Lock(); defer eng.mu.Unlock(); return len(eng.hibernates) == 1 }) + eng.mu.Lock() + syncedBeforeCapture := slices.Clone(eng.syncs) + eng.mu.Unlock() + close(eng.hibernateStall) + <-hibernated + + want := []string(nil) + if on { + want = []string{sb.VsockSocket} + } + if !slices.Equal(syncedBeforeCapture, want) { + t.Errorf("synced %v before the capture, want %v", syncedBeforeCapture, want) + } + }) + } +} + +func TestReconcileClearsReaderLeftovers(t *testing.T) { + eng := newFakeEngine() + m := newReaderManager(t, eng) + stale := filepath.Join(m.dataDir, readerDir, "sb_old-aabbcc") + if err := os.MkdirAll(stale, 0o750); err != nil { + t.Fatalf("setup: %v", err) + } + if err := m.Reconcile(t.Context()); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if _, err := os.Stat(stale); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a reader dir from a previous run survived the restart: %v", err) + } +} + +func newReaderManager(t *testing.T, eng *fakeEngine) *Manager { + t.Helper() + cfg := &config.Config{ + DataDir: t.TempDir(), + Reader: &config.ReaderSpec{Template: readerKey.Template}, + Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}, {PoolKey: readerKey, Warm: 1}}, + } + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatalf("setup manager: %v", err) + } + return m +} + +func seedReaderVMs(m *Manager, names ...string) { + for _, name := range names { + m.pools[readerKey].warm = append(m.pools[readerKey].warm, &types.Sandbox{VMName: name, Key: readerKey, VsockSocket: "/vsock/" + name}) + } +} + +func hibernateClaim(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } +} + +// readerExport makes every snapshot export carry the disk, a sidecar naming two layers, and a memory image a reader must drop. +func readerExport(t *testing.T, eng *fakeEngine) []string { + t.Helper() + dir := t.TempDir() + layers := []string{filepath.Join(dir, "base.erofs"), filepath.Join(dir, "top.erofs")} + for _, layer := range layers { + if err := os.WriteFile(layer, []byte("erofs"), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + } + sidecar := fmt.Sprintf(`{"storage_configs":[{"path":%q,"role":"layer"},{"path":%q,"role":"layer"},{"path":"/run/cow.raw","role":"cow"},{"path":"/run/cidata.img","role":"cidata"}]}`, layers[0], layers[1]) + eng.exportFiles = map[string]string{"cow.raw": "disk-bytes", "cocoon.json": sidecar, "memory-ranges": "guest-memory"} + return layers +} + +func readerOps(eng *fakeEngine) []string { + eng.mu.Lock() + defer eng.mu.Unlock() + return slices.DeleteFunc(slices.Clone(eng.volumeOps), func(op string) bool { + return !strings.HasPrefix(op, "attach:") && !strings.HasPrefix(op, "reader-start:") + }) +} + +func readerLifecycle(eng *fakeEngine) (removes []string, setups int) { + eng.mu.Lock() + defer eng.mu.Unlock() + return slices.Clone(eng.removes), len(eng.startedReaders) +} + +type fetchCountingStore struct { + store.Store + mu sync.Mutex + fetches int + fetched []string +} + +func (s *fetchCountingStore) Fetch(ctx context.Context, id string) (string, []byte, string, error) { + s.mu.Lock() + s.fetches++ + s.mu.Unlock() + return s.Store.Fetch(ctx, id) +} + +func (s *fetchCountingStore) FetchFile(ctx context.Context, id, rel string) (string, error) { + s.mu.Lock() + s.fetched = append(s.fetched, rel) + s.mu.Unlock() + return s.Store.FetchFile(ctx, id, rel) +} + +func (s *fetchCountingStore) files() []string { + s.mu.Lock() + defer s.mu.Unlock() + out := slices.Clone(s.fetched) + slices.Sort(out) + return out +} diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 0905a93c..1cf4ba80 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -100,6 +100,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { } // a crash mid-export leaves a fork-* staging dir nothing in this life reuses _ = utils.RemoveDirEntries(m.dataDir, func(name string) bool { return strings.HasPrefix(name, "fork-") }) + _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, readerDir), nil) if fenced { m.reclaimOrphanArchiveCks(ctx, claims) From b71674983789a9093924cf586e1e0ac9a73cda25 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 02:01:30 +0800 Subject: [PATCH 07/28] server: serve a sleeping claim's read-only calls from its reader A connection to a hibernated or archived claim gets its 101 first, so the opening frame can be read before the wake is chosen. Read-only calls (fs_stat, fs_list, fs_read, fs_pull) go to the claim's reader; an info frame pipelined ahead of one rides along. The first other call, or a claim that woke or slept again, hands the same connection to the woken guest with the unread bytes replayed, so a parked SDK connection never sees a close. --- e2e/reader_test.go | 76 +++++++ sandboxd/server/exec.go | 2 +- sandboxd/server/reader.go | 291 ++++++++++++++++++++++++ sandboxd/server/reader_test.go | 402 +++++++++++++++++++++++++++++++++ sandboxd/server/relay.go | 95 +++++--- sandboxd/server/relay_test.go | 6 +- sandboxd/server/server.go | 7 +- sandboxd/server/server_http.go | 22 +- sandboxd/server/server_test.go | 25 ++ 9 files changed, 878 insertions(+), 48 deletions(-) create mode 100644 e2e/reader_test.go create mode 100644 sandboxd/server/reader.go create mode 100644 sandboxd/server/reader_test.go diff --git a/e2e/reader_test.go b/e2e/reader_test.go new file mode 100644 index 00000000..185cce61 --- /dev/null +++ b/e2e/reader_test.go @@ -0,0 +1,76 @@ +package e2e + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +var readerKey = types.PoolKey{Template: "reader:24.04", Net: types.NetNone, Size: types.SizeSmall} + +func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { + st := startConfigStack(t, "node-token", &config.Config{ + Reader: &config.ReaderSpec{Template: readerKey.Template}, + Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}, {PoolKey: readerKey, Warm: 1}}, + }) + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "work"), 0o750); err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "work", "out.txt"), []byte("result"), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + st.eng.mu.Lock() + st.eng.readerRoot = root + st.eng.mu.Unlock() + waitFor(t, func() bool { + pools, _ := st.mgr.Info() + return slices.ContainsFunc(pools, func(p pool.PoolInfo) bool { return p.Key == readerKey && p.Warm >= 1 }) + }) + + sb, err := st.client.New(t.Context(), testKey.Template) + if err != nil { + t.Fatalf("New: %v", err) + } + defer func() { _ = sb.Close() }() + if err = sb.Hibernate(t.Context()); err != nil { + t.Fatalf("Hibernate: %v", err) + } + + got, err := sb.ReadFile(t.Context(), "/work/out.txt") + if err != nil || string(got) != "result" { + t.Fatalf("ReadFile of a hibernated claim = %q, %v, want the reader's file", got, err) + } + if !hibernated(t, st, sb.ID) { + t.Fatal("reading a file woke the claim") + } + st.eng.mu.Lock() + started := len(st.eng.readerStarts) + st.eng.mu.Unlock() + if started != 1 { + t.Errorf("%d readers started, want one", started) + } + + if out, execErr := sb.Exec(t.Context(), "echo", "woke"); execErr != nil || out != "woke\n" { + t.Fatalf("Exec on the sleeping claim = %q, %v, want it to wake and answer", out, execErr) + } + if hibernated(t, st, sb.ID) { + t.Error("an exec left the claim hibernated") + } +} + +func hibernated(t *testing.T, st *stack, id string) bool { + t.Helper() + for _, s := range st.mgr.Sandboxes("", "", nil) { + if s.ID == id { + return s.Hibernated + } + } + t.Fatalf("claim %s not listed", id) + return false +} diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go index 1f432613..470100e1 100644 --- a/sandboxd/server/exec.go +++ b/sandboxd/server/exec.go @@ -61,7 +61,7 @@ func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { } ctx := r.Context() id := r.PathValue("id") - guest, done, ok := s.wakeGuest(ctx, w, id, token) + guest, done, _, ok := s.wakeGuest(ctx, w, id, token, s.mgr.WakeAgentSocket) if !ok { return } diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go new file mode 100644 index 00000000..b82f57ea --- /dev/null +++ b/sandboxd/server/reader.go @@ -0,0 +1,291 @@ +package server + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "slices" + "sync/atomic" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/protocol/wire" + "github.com/cocoonstack/sandbox/sandboxd/pool" +) + +const ( + // infoWait bounds how long a lone info frame waits for the call a client pipelines behind it. + infoWait = 100 * time.Millisecond + routeBuf = 64 << 10 +) + +var ( + readOnlyOps = map[string]bool{ + wire.FsStat{}.Op(): true, + wire.FsList{}.Op(): true, + wire.FsRead{}.Op(): true, + wire.FsPull{}.Op(): true, + } + infoOp = wire.Info{}.Op() + + errReaderGone = errors.New("reader connection lost") + errSleepEnded = errors.New("sandbox is no longer in the sleep the reader serves") + errUnreachable = errors.New("guest agent unreachable") +) + +// serveSleeping answers the 101 before reading any frame, since a client's first frame can wait on it. +func (s *Server) serveSleeping(ctx context.Context, w http.ResponseWriter, id, token string) { + client, clientBuf, ok := hijack(ctx, w) + if !ok { + return + } + release, ok := s.trackRelay(client, noCloser{}) + if !ok { + return + } + defer release() + _ = client.SetDeadline(time.Time{}) + keepAlive(client) + if _, err := client.Write(switchingProtocols); err != nil { + return + } + rt := &router{s: s, ctx: ctx, id: id, token: token, client: client, br: bufio.NewReaderSize(clientReader(client, clientBuf), routeBuf)} + guest, done, ok := rt.run() + if !ok { + return + } + defer done() + defer func() { _ = guest.Close() }() + s.retargetRelay(client, guest) + copyBoth(client, s.audited(ctx, id, io.MultiReader(bytes.NewReader(rt.replay), rt.br)), guest, silkdEnds) +} + +func (s *Server) openGuest(ctx context.Context, id, token string) (net.Conn, func(), error) { + sock, _, done, err := s.mgr.WakeAgentSocket(ctx, id, token) + if err != nil { + return nil, nil, err + } + guest, err := s.dialer.DialSilkd(ctx, sock) + if err != nil { + done() + return nil, nil, fmt.Errorf("%w: %w", errUnreachable, err) + } + return guest, done, nil +} + +type router struct { + s *Server + ctx context.Context + id string + token string + + client net.Conn + br *bufio.Reader + + reader net.Conn + source string + pumped chan struct{} + superseded atomic.Bool + + pending []byte + // replay is what the guest has not seen when the router hands over. + replay []byte +} + +func (r *router) run() (net.Conn, func(), bool) { + defer r.closeReader() + for { + head, partial, err := r.readHead() + if err != nil { + return nil, nil, false + } + op := wire.RequestOp(head) + switch { + case wire.IsContinuation(head): + if r.reader == nil { + err = r.pass(head, partial, io.Discard) + } else { + err = r.pass(head, partial, r.reader) + } + if err != nil { + return nil, nil, false + } + case op == infoOp && r.reader == nil: + r.pending = append(r.pending, head...) + followed, waitErr := r.infoFollowed() + if waitErr != nil { + return nil, nil, false + } + if !followed { + return r.handover(nil) + } + case readOnlyOps[op] || op == infoOp: + switch err = r.toReader(head, partial); { + case err == nil: + case errors.Is(err, errReaderGone): + return nil, nil, false + case errors.Is(err, pool.ErrUnknownSandbox): + r.fail(err) + return nil, nil, false + default: + return r.handover(head) + } + default: + return r.handover(head) + } + } +} + +func (r *router) toReader(head []byte, partial bool) error { + if r.reader == nil { + if err := r.dialReader(); err != nil { + return err + } + } else if r.s.mgr.SleepingSource(r.id, r.token) != r.source { + return errSleepEnded + } + if r.s.mgr.AuditEnabled() { + r.audit(head, partial) + } + if err := r.pass(head, partial, r.reader); err != nil { + return errReaderGone + } + return nil +} + +func (r *router) dialReader() error { + conn, source, err := r.s.mgr.DialReader(r.ctx, r.id, r.token) + if err != nil { + return err + } + r.reader, r.source = conn, source + r.startPump() + if len(r.pending) == 0 { + return nil + } + if r.s.mgr.AuditEnabled() { + r.auditPending() + } + if _, err = conn.Write(r.pending); err != nil { + return errReaderGone + } + r.pending = nil + return nil +} + +func (r *router) handover(head []byte) (net.Conn, func(), bool) { + r.closeReader() + guest, done, err := r.s.openGuest(r.ctx, r.id, r.token) + if err != nil { + r.fail(err) + return nil, nil, false + } + r.replay = slices.Concat(r.pending, head) + return guest, done, true +} + +func (r *router) readHead() (head []byte, partial bool, err error) { + head, err = r.br.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + return head, true, nil + } + return head, false, err +} + +func (r *router) pass(head []byte, partial bool, w io.Writer) error { + if _, err := w.Write(head); err != nil { + return err + } + for partial { + next, err := r.br.ReadSlice('\n') + partial = errors.Is(err, bufio.ErrBufferFull) + if err != nil && !partial { + return err + } + if _, err = w.Write(next); err != nil { + return err + } + } + return nil +} + +func (r *router) infoFollowed() (bool, error) { + if r.br.Buffered() > 0 { + return true, nil + } + _ = r.client.SetReadDeadline(time.Now().Add(infoWait)) + _, err := r.br.Peek(1) + _ = r.client.SetReadDeadline(time.Time{}) + if timeout, ok := errors.AsType[net.Error](err); ok && timeout.Timeout() { + return false, nil + } + return err == nil, err +} + +func (r *router) startPump() { + r.pumped = make(chan struct{}) + reader := r.reader + go func() { + defer close(r.pumped) + _, _ = io.Copy(r.client, reader) + if !r.superseded.Load() { + _ = r.client.Close() + } + }() +} + +func (r *router) closeReader() { + if r.reader == nil { + return + } + r.superseded.Store(true) + _ = r.reader.Close() + <-r.pumped + r.reader = nil +} + +// fail answers with an error frame, the only reply left once the 101 has gone out. +func (r *router) fail(err error) { + kind, msg := wire.KindInternal, "sandbox lookup failed" + switch _, mapped, ok := lookupPoolErr(err); { + case errors.Is(err, errUnreachable): + msg = errUnreachable.Error() + log.WithFunc("server.fail").Errorf(r.ctx, err, "dial silkd for %s", r.id) + case ok: + msg = mapped + if errors.Is(err, pool.ErrUnknownSandbox) { + kind = wire.KindNotFound + } + default: + log.WithFunc("server.fail").Errorf(r.ctx, err, "agent socket for %s", r.id) + } + frame, encodeErr := wire.EncodeResponse(&wire.ErrorResp{Kind: kind, Message: msg}) + if encodeErr != nil { + return + } + _, _ = r.client.Write(append(frame, '\n')) +} + +func (r *router) auditPending() { + for line := range bytes.SplitSeq(bytes.TrimSuffix(r.pending, newline), newline) { + r.s.mgr.Audit(r.ctx, r.id, line) + } +} + +func (r *router) audit(head []byte, partial bool) { + if partial { + r.s.mgr.Audit(r.ctx, r.id, head) + return + } + r.s.mgr.Audit(r.ctx, r.id, bytes.TrimSuffix(head, newline)) +} + +type noCloser struct{} + +func (noCloser) Close() error { return nil } diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go new file mode 100644 index 00000000..fb66a70f --- /dev/null +++ b/sandboxd/server/reader_test.go @@ -0,0 +1,402 @@ +package server + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "net/http/httptest" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cocoonstack/sandbox/protocol/wire" + "github.com/cocoonstack/sandbox/sandboxd/pool" +) + +var ( + infoFrame = line(wire.Info{}) + readFrame = line(wire.FsRead{Path: "/work/out.txt"}) +) + +func TestSleepingReadIsServedByTheReaderWithoutWaking(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + + send(t, conn, statFrame) + expectLine(t, r, "reader") + if wakes, dials := env.reader.counts(); wakes != 0 || dials != 1 { + t.Errorf("wakes=%d reader dials=%d, want the read served by one reader and no wake", wakes, dials) + } + if guests := env.guestDials.Load(); guests != 0 { + t.Errorf("%d guest dials, want none", guests) + } + if got := env.frames.all(); !slices.Equal(got, []string{"reader:" + statFrame}) { + t.Errorf("frames %q, want the stat at the reader only", got) + } +} + +func TestSleepingFirstDialPipelinesInfoBehindTheRead(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + + send(t, conn, infoFrame) + time.Sleep(10 * time.Millisecond) + send(t, conn, readFrame) + expectLine(t, r, "reader") + expectLine(t, r, "reader") + if wakes, _ := env.reader.counts(); wakes != 0 { + t.Errorf("wakes=%d, want the info and the read served by the reader", wakes) + } + if got := env.frames.all(); !slices.Equal(got, []string{"reader:" + infoFrame, "reader:" + readFrame}) { + t.Errorf("frames %q, want info then read at the reader in order", got) + } +} + +func TestSleepingLoneInfoWakes(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + + send(t, conn, infoFrame) + expectLine(t, r, "guest") + if wakes, dials := env.reader.counts(); wakes != 1 || dials != 0 { + t.Errorf("wakes=%d reader dials=%d, want a wake and no reader", wakes, dials) + } + if got := env.frames.all(); !slices.Equal(got, []string{"guest:" + infoFrame}) { + t.Errorf("frames %q, want the info replayed to the guest", got) + } +} + +func TestSleepingFirstWriteWakesAsBefore(t *testing.T) { + for name, first := range map[string]string{"exec": execFrame, "write": line(wire.FsWrite{Path: "/x"}), "port": line(wire.PortForward{Port: 80})} { + t.Run(name, func(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + + send(t, conn, first) + expectLine(t, r, "guest") + if wakes, dials := env.reader.counts(); wakes != 1 || dials != 0 { + t.Errorf("wakes=%d reader dials=%d, want a wake and no reader", wakes, dials) + } + if got := env.frames.all(); !slices.Equal(got, []string{"guest:" + first}) { + t.Errorf("frames %q, want the first call at the guest", got) + } + }) + } +} + +func TestSleepingLaterWriteMovesTheConnectionToTheGuest(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + send(t, conn, statFrame) + expectLine(t, r, "reader") + + send(t, conn, execFrame) + expectLine(t, r, "guest") + select { + case <-env.readerClosed: + case <-time.After(3 * time.Second): + t.Fatal("the reader connection stayed open after the claim woke") + } + send(t, conn, statFrame) + expectLine(t, r, "guest") + if wakes, dials := env.reader.counts(); wakes != 1 || dials != 1 { + t.Errorf("wakes=%d reader dials=%d, want one reader read then one wake", wakes, dials) + } + want := []string{"reader:" + statFrame, "guest:" + execFrame, "guest:" + statFrame} + if got := env.frames.all(); !slices.Equal(got, want) { + t.Errorf("frames %q, want %q", got, want) + } +} + +func TestSleepingReaderStopsServingOnceTheClaimWokeOrSleptAgain(t *testing.T) { + for name, next := range map[string]string{"woke": "", "slept again": "snap-2"} { + t.Run(name, func(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + send(t, conn, statFrame) + expectLine(t, r, "reader") + + env.reader.setSource(next) + send(t, conn, statFrame) + expectLine(t, r, "guest") + select { + case <-env.readerClosed: + case <-time.After(3 * time.Second): + t.Fatal("the stale reader connection stayed open") + } + }) + } +} + +func TestSleepingUnavailableReaderWakes(t *testing.T) { + env := newSleepingEnv(t) + env.reader.setDialErr(pool.ErrNoReader) + conn, r := env.upgrade(t) + + send(t, conn, statFrame) + expectLine(t, r, "guest") + if wakes, _ := env.reader.counts(); wakes != 1 { + t.Errorf("wakes=%d, want the read to fall back to a wake", wakes) + } +} + +func TestSleepingFailuresAnswerWithAnErrorFrame(t *testing.T) { + for _, tt := range []struct { + name string + dialErr error + wakeErr error + firstCall string + kind string + message string + }{ + {"unknown sandbox", pool.ErrUnknownSandbox, nil, statFrame, wire.KindNotFound, "unknown sandbox"}, + {"wake refused", pool.ErrNoReader, pool.ErrTenantRemoved, statFrame, wire.KindInternal, pool.ErrTenantRemoved.Error()}, + {"wake fails", nil, errors.New("restore broke"), execFrame, wire.KindInternal, "sandbox lookup failed"}, + } { + t.Run(tt.name, func(t *testing.T) { + env := newSleepingEnv(t) + env.reader.setDialErr(tt.dialErr) + env.manager.socket = func(string, string) (string, error) { return "", tt.wakeErr } + conn, r := env.upgrade(t) + + send(t, conn, tt.firstCall) + reply, err := r.ReadString('\n') + if err != nil { + t.Fatalf("read reply: %v", err) + } + resp, err := wire.DecodeResponse([]byte(reply)) + if err != nil { + t.Fatalf("decode %q: %v", reply, err) + } + if got, ok := resp.(*wire.ErrorResp); !ok || got.Kind != tt.kind || got.Message != tt.message { + t.Errorf("reply %+v, want an error frame of kind %s saying %q", resp, tt.kind, tt.message) + } + if _, err = r.ReadString('\n'); !errors.Is(err, io.EOF) { + t.Errorf("connection after the error frame: %v, want EOF", err) + } + }) + } +} + +func TestSleepingFramesAreAuditedOnceOnEitherPath(t *testing.T) { + env := newSleepingEnv(t) + var mu sync.Mutex + var audited []string + env.manager.audited = func(_ string, frame []byte) { + mu.Lock() + audited = append(audited, string(frame)) + mu.Unlock() + } + conn, r := env.upgrade(t) + send(t, conn, infoFrame) + time.Sleep(10 * time.Millisecond) + send(t, conn, statFrame) + expectLine(t, r, "reader") + expectLine(t, r, "reader") + send(t, conn, execFrame) + expectLine(t, r, "guest") + + mu.Lock() + defer mu.Unlock() + want := []string{strings.TrimSpace(infoFrame), strings.TrimSpace(statFrame), strings.TrimSpace(execFrame)} + if !slices.Equal(audited, want) { + t.Errorf("audited %q, want each opening frame once: %q", audited, want) + } +} + +func TestSleepingLongReadFrameReachesTheReaderIntact(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + long := line(wire.FsRead{Path: "/" + strings.Repeat("a", 3*routeBuf)}) + + send(t, conn, long) + expectLine(t, r, "reader") + if got := env.frames.all(); !slices.Equal(got, []string{"reader:" + long}) { + t.Errorf("the reader received %d frames, want the long frame whole", len(got)) + } +} + +func TestSleepingAwakeClaimRelaysToItsGuest(t *testing.T) { + env := newSleepingEnv(t) + env.reader.setSource("") + conn, r := env.upgrade(t) + + send(t, conn, statFrame) + expectLine(t, r, "guest") + if _, dials := env.reader.counts(); dials != 0 { + t.Errorf("reader dials=%d for a running claim, want none", dials) + } +} + +func TestSleepingCloseRelaysEndsReaderConnections(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + send(t, conn, statFrame) + expectLine(t, r, "reader") + + done := make(chan struct{}) + go func() { + env.srv.CloseRelays() + close(done) + }() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("CloseRelays hung on a reader connection") + } + if _, err := r.ReadString('\n'); !errors.Is(err, io.EOF) { + t.Errorf("client after CloseRelays: %v, want EOF", err) + } +} + +type sleepingEnv struct { + ts *httptest.Server + srv *Server + manager *fakeManager + reader *fakeReader + frames *frameLog + guestDials *atomic.Int32 + readerClosed chan struct{} +} + +func newSleepingEnv(t *testing.T) *sleepingEnv { + t.Helper() + env := &sleepingEnv{frames: &frameLog{}, guestDials: &atomic.Int32{}, readerClosed: make(chan struct{}, 4)} + env.reader = &fakeReader{src: "snap-1", serve: tagServer("reader", env.frames, env.readerClosed)} + env.manager = &fakeManager{reader: env.reader} + dialer := &fakeDialer{dial: func(context.Context, string) (net.Conn, error) { + env.guestDials.Add(1) + relayEnd, guestEnd := net.Pipe() + go tagServer("guest", env.frames, nil)(guestEnd) + return relayEnd, nil + }} + env.srv = New("", "node:7777", env.manager, dialer, nil, nil, nil, nil) + env.ts = httptest.NewServer(env.srv.Handler()) + t.Cleanup(func() { + env.srv.CloseRelays() + env.ts.Close() + }) + return env +} + +func (e *sleepingEnv) upgrade(t *testing.T) (net.Conn, *bufio.Reader) { + t.Helper() + return upgradeConn(t, e.ts) +} + +func send(t *testing.T, conn net.Conn, frame string) { + t.Helper() + if _, err := io.WriteString(conn, frame); err != nil { + t.Fatalf("write frame: %v", err) + } +} + +func expectLine(t *testing.T, r *bufio.Reader, want string) { + t.Helper() + got, err := r.ReadString('\n') + if err != nil { + t.Fatalf("read reply (want %q): %v", want, err) + } + if strings.TrimSpace(got) != want { + t.Fatalf("reply %q, want %q", got, want) + } +} + +func tagServer(tag string, frames *frameLog, closed chan<- struct{}) func(net.Conn) { + return func(c net.Conn) { + defer func() { + _ = c.Close() + if closed != nil { + select { + case closed <- struct{}{}: + default: + } + } + }() + r := bufio.NewReaderSize(c, 1<<20) + for { + got, err := r.ReadString('\n') + if err != nil { + return + } + frames.add(tag + ":" + got) + if _, err = io.WriteString(c, tag+"\n"); err != nil { + return + } + } + } +} + +type frameLog struct { + mu sync.Mutex + entries []string +} + +func (l *frameLog) add(entry string) { + l.mu.Lock() + defer l.mu.Unlock() + l.entries = append(l.entries, entry) +} + +func (l *frameLog) all() []string { + l.mu.Lock() + defer l.mu.Unlock() + return slices.Clone(l.entries) +} + +type fakeReader struct { + mu sync.Mutex + src string + wakes int + dials int + dialErr error + serve func(net.Conn) +} + +func (f *fakeReader) source() string { + f.mu.Lock() + defer f.mu.Unlock() + return f.src +} + +func (f *fakeReader) setSource(src string) { + f.mu.Lock() + defer f.mu.Unlock() + f.src = src +} + +func (f *fakeReader) setDialErr(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.dialErr = err +} + +func (f *fakeReader) wake() { + f.mu.Lock() + defer f.mu.Unlock() + f.src = "" + f.wakes++ +} + +func (f *fakeReader) counts() (wakes, dials int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.wakes, f.dials +} + +func (f *fakeReader) dial() (net.Conn, string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.dialErr != nil { + return nil, "", f.dialErr + } + f.dials++ + relayEnd, readerEnd := net.Pipe() + go f.serve(readerEnd) + return relayEnd, f.src, nil +} diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 865feabf..9cf4c7a6 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -35,6 +35,8 @@ var ( silkdEnds = relayEnds{guest: func(c net.Conn) { _ = c.Close() }, client: drainClient} ) +type agentWaker func(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) + // relayEnds is what a finished direction means to the peer that did not end it. type relayEnds struct { guest, client func(net.Conn) @@ -57,7 +59,12 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { if !ok { return } - guest, done, ok := s.wakeGuest(r.Context(), w, r.PathValue("id"), token) + id := r.PathValue("id") + guest, done, sleeping, ok := s.wakeGuest(r.Context(), w, id, token, s.mgr.WakeAgentSocketOrRead) + if sleeping { + s.serveSleeping(r.Context(), w, id, token) + return + } if !ok { return } @@ -66,19 +73,21 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { if !ok { return } - s.relay(r.Context(), r.PathValue("id"), client, clientBuf, guest) + s.relay(r.Context(), id, client, clientBuf, guest) } -// wakeGuest dials the sandbox's silkd, waking a hibernated VM first; on failure it has already answered the request. -func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token string) (guest net.Conn, release func(), ok bool) { - sock, fenced, done, err := s.mgr.WakeAgentSocket(ctx, id, token) +// wakeGuest dials the sandbox's silkd, waking a hibernated VM first unless wake answers ErrSleeping; on failure it has already answered the request. +func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token string, wake agentWaker) (guest net.Conn, release func(), sleeping, ok bool) { + sock, fenced, done, err := wake(ctx, id, token) switch { + case errors.Is(err, pool.ErrSleeping): + return nil, nil, true, false case writePoolErr(w, err): - return nil, nil, false + return nil, nil, false, false case err != nil: log.WithFunc("server.wakeGuest").Errorf(ctx, err, "agent socket for %s", id) writeErr(w, http.StatusInternalServerError, "sandbox lookup failed") - return nil, nil, false + return nil, nil, false, false } guest, err = s.dialer.DialSilkd(ctx, sock) if err != nil { @@ -89,23 +98,26 @@ func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token log.WithFunc("server.wakeGuest").Errorf(ctx, err, "dial silkd for %s", id) } writeErr(w, http.StatusBadGateway, "guest agent unreachable") - return nil, nil, false + return nil, nil, false, false } if fenced == nil { - return guest, done, true + return guest, done, false, true } stop := pool.CloseOnFence(fenced, guest) - return guest, func() { stop(); done() }, true + return guest, func() { stop(); done() }, false, true } func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBuf *bufio.Reader, guest net.Conn) { - clientR := clientReader(client, clientBuf) - if s.mgr.AuditEnabled() { - clientR = &auditTee{r: clientR, record: func(line []byte) { - s.mgr.Audit(ctx, id, line) - }} + s.splice(client, s.audited(ctx, id, clientReader(client, clientBuf)), guest, switchingProtocols, silkdEnds) +} + +func (s *Server) audited(ctx context.Context, id string, clientR io.Reader) io.Reader { + if !s.mgr.AuditEnabled() { + return clientR } - s.splice(client, clientR, guest, switchingProtocols, silkdEnds) + return &auditTee{r: clientR, record: func(line []byte) { + s.mgr.Audit(ctx, id, line) + }} } func (s *Server) splice(client net.Conn, clientR io.Reader, guest net.Conn, hello []byte, ends relayEnds) { @@ -121,30 +133,11 @@ func (s *Server) splice(client net.Conn, clientR io.Reader, guest net.Conn, hell if _, err := client.Write(hello); err != nil { return } - - done := make(chan struct{}) - go func() { - defer close(done) - _, err := io.Copy(guest, clientR) - if err != nil { - // the client died rather than half-closing, so nothing is left to answer it - _ = guest.Close() - return - } - ends.guest(guest) - }() - - if _, err := io.Copy(client, guest); err != nil { - _ = client.Close() - _ = guest.Close() - } else { - ends.client(client) - } - <-done + copyBoth(client, clientR, guest, ends) } // trackRelay registers a hijacked pair for CloseRelays; !ok means the server is already draining. -func (s *Server) trackRelay(client, guest net.Conn) (func(), bool) { +func (s *Server) trackRelay(client net.Conn, guest io.Closer) (func(), bool) { s.relayMu.Lock() if s.relayClosed { s.relayMu.Unlock() @@ -165,6 +158,12 @@ func (s *Server) trackRelay(client, guest net.Conn) (func(), bool) { }, true } +func (s *Server) retargetRelay(client net.Conn, peer io.Closer) { + s.relayMu.Lock() + s.relays[client] = peer + s.relayMu.Unlock() +} + // auditTee records each request line it relays; input frames and the bytes past the cap pass unrecorded. type auditTee struct { r io.Reader @@ -212,6 +211,28 @@ func (t *auditTee) endLine() { t.skip = false } +func copyBoth(client net.Conn, clientR io.Reader, guest net.Conn, ends relayEnds) { + done := make(chan struct{}) + go func() { + defer close(done) + _, err := io.Copy(guest, clientR) + if err != nil { + // the client died rather than half-closing, so nothing is left to answer it + _ = guest.Close() + return + } + ends.guest(guest) + }() + + if _, err := io.Copy(client, guest); err != nil { + _ = client.Close() + _ = guest.Close() + } else { + ends.client(client) + } + <-done +} + func keepAlive(client net.Conn) { tcp, ok := client.(*net.TCPConn) if !ok { diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 0c0b5aef..23490d69 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -184,7 +184,7 @@ func TestWakeGuestLogsAClientCancelBelowError(t *testing.T) { canceled, cancel := context.WithCancel(t.Context()) cancel() w := httptest.NewRecorder() - if _, _, ok := srv.wakeGuest(canceled, w, "sb_1", "tok"); ok || w.Code != http.StatusBadGateway { + if _, _, _, ok := srv.wakeGuest(canceled, w, "sb_1", "tok", srv.mgr.WakeAgentSocket); ok || w.Code != http.StatusBadGateway { t.Fatalf("canceled dial: ok %v status %d, want a refused 502", ok, w.Code) } if logs.Len() != 0 { @@ -192,7 +192,7 @@ func TestWakeGuestLogsAClientCancelBelowError(t *testing.T) { } dialer.dial = func(context.Context, string) (net.Conn, error) { return nil, errors.New("connection refused") } - if _, _, ok := srv.wakeGuest(t.Context(), httptest.NewRecorder(), "sb_1", "tok"); ok { + if _, _, _, ok := srv.wakeGuest(t.Context(), httptest.NewRecorder(), "sb_1", "tok", srv.mgr.WakeAgentSocket); ok { t.Fatal("a failing dial succeeded") } if !strings.Contains(logs.String(), `"level":"error"`) { @@ -201,7 +201,7 @@ func TestWakeGuestLogsAClientCancelBelowError(t *testing.T) { logs.Reset() dialer.dial = func(context.Context, string) (net.Conn, error) { return nil, context.Canceled } - if _, _, ok := srv.wakeGuest(t.Context(), httptest.NewRecorder(), "sb_1", "tok"); ok { + if _, _, _, ok := srv.wakeGuest(t.Context(), httptest.NewRecorder(), "sb_1", "tok", srv.mgr.WakeAgentSocket); ok { t.Fatal("a failing dial succeeded") } if !strings.Contains(logs.String(), `"level":"error"`) { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 3521dedd..78b44fd6 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -123,6 +123,9 @@ type Manager interface { AdoptArchived(ctx context.Context, id, token string) (bool, error) DialPort(ctx context.Context, id string, cred pool.Cred, port uint16) (net.Conn, error) WakeAgentSocket(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) + WakeAgentSocketOrRead(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) + SleepingSource(id, token string) string + DialReader(ctx context.Context, id, token string) (conn net.Conn, source string, err error) SetPools(ctx context.Context, pools []config.PoolSpec) error TenantByToken(ctx context.Context, token string) (*tenants.TenantRecord, error) Tenants(ctx context.Context, after string, limit int) (pool.TenantPage, error) @@ -229,7 +232,7 @@ type Server struct { execBudget *semaphore.Weighted relayMu sync.Mutex - relays map[net.Conn]net.Conn // client conn → guest conn + relays map[net.Conn]io.Closer // client conn → what its relay holds open relayClosed bool relayWG sync.WaitGroup } @@ -250,7 +253,7 @@ func New(apiToken, advertise string, mgr Manager, dialer Dialer, placer Placer, apiToken: apiToken, advertise: advertise, preview: preview, - relays: map[net.Conn]net.Conn{}, + relays: map[net.Conn]io.Closer{}, execBudget: semaphore.NewWeighted(execBufferBudget), } diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index d148bf35..48bf2a94 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -89,10 +89,17 @@ func upgradeGate(w http.ResponseWriter, r *http.Request, proto string) (string, // hijackClient on failure has already closed guest and answered the request. func hijackClient(ctx context.Context, w http.ResponseWriter, guest net.Conn) (net.Conn, *bufio.Reader, bool) { + client, buf, ok := hijack(ctx, w) + if !ok { + _ = guest.Close() + } + return client, buf, ok +} + +func hijack(ctx context.Context, w http.ResponseWriter) (net.Conn, *bufio.Reader, bool) { client, bufrw, err := http.NewResponseController(w).Hijack() if err != nil { - _ = guest.Close() - log.WithFunc("server.hijackClient").Error(ctx, err, "hijack") + log.WithFunc("server.hijack").Error(ctx, err, "hijack") writeErr(w, http.StatusInternalServerError, "connection cannot be hijacked") return nil, nil, false } @@ -171,11 +178,16 @@ func writePoolErr(w http.ResponseWriter, err error) bool { } func mapPoolErr(w http.ResponseWriter, err error) (code int, msg string, ok bool) { + code, msg, ok = lookupPoolErr(err) + if ok && code == http.StatusServiceUnavailable { + w.Header().Set("Retry-After", retryAfterSeconds) + } + return code, msg, ok +} + +func lookupPoolErr(err error) (code int, msg string, ok bool) { for _, m := range poolErrHTTP { if errors.Is(err, m.err) { - if m.code == http.StatusServiceUnavailable { - w.Header().Set("Retry-After", retryAfterSeconds) - } return m.code, cmp.Or(m.msg, err.Error()), true } } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 4fb76cb8..6debba66 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2554,6 +2554,7 @@ type fakeManager struct { sandboxIndex func(tenant, claimRef string) []pool.SandboxSummary sandboxByID func(id string) (pool.SandboxSummary, bool) draining bool + reader *fakeReader } func (f *fakeManager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { @@ -2617,10 +2618,34 @@ func (f *fakeManager) AdoptArchived(_ context.Context, id, token string) (bool, } func (f *fakeManager) WakeAgentSocket(_ context.Context, id, token string) (string, <-chan struct{}, func(), error) { + if f.reader != nil { + f.reader.wake() + } sock, err := f.AgentSocket(id, token) return sock, f.fenced, func() {}, err } +func (f *fakeManager) WakeAgentSocketOrRead(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) { + if f.reader != nil && f.reader.source() != "" { + return "", nil, nil, pool.ErrSleeping + } + return f.WakeAgentSocket(ctx, id, token) +} + +func (f *fakeManager) SleepingSource(_, _ string) string { + if f.reader == nil { + return "" + } + return f.reader.source() +} + +func (f *fakeManager) DialReader(_ context.Context, _, _ string) (net.Conn, string, error) { + if f.reader == nil { + return nil, "", pool.ErrNoReader + } + return f.reader.dial() +} + func (f *fakeManager) Renew(_ context.Context, id string, cred pool.Cred, ttl time.Duration, onExpire types.ExpireAction) (time.Time, error) { f.gotOnExpire = onExpire if f.renew == nil { From d25134abbce9a1e8e0ee9c36e39e09e6409e3498 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 02:02:38 +0800 Subject: [PATCH 08/28] docs: reader config and the sleeping-claim relay behavior --- README.md | 4 ++- docs/deploy.md | 62 ++++++++++++++++++++++++++++++++++++++++++++ docs/sandboxd-api.md | 23 +++++++++++++++- docs/sdk.md | 5 ++++ docs/silkd.md | 5 +++- 5 files changed, 96 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa3f9d1b..13361755 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ performance) — source in snapshot exports (online-retunable), claim/release/renew/hibernate/fork/ promote/checkpoint HTTP API, operator-catalog dataset volumes (read-only or writable; read-only ones change on a running claim), signed preview URLs, - the HTTP-upgrade byte relays to silkd and to any guest port, claim env with + the HTTP-upgrade byte relays to silkd and to any guest port, read-only file + access to hibernated or archived claims from a disposable reader VM + (opt-in `reader`), claim env with host-side credential injection and per-claim upstream proxies, runtime tenants, config reload (SIGHUP), usage + audit journals, /metrics, reap + restart reconcile with VMM cold restart, memberlist mesh with redirect diff --git a/docs/deploy.md b/docs/deploy.md index 5d835c82..c52dbc1b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -148,6 +148,7 @@ gets the intersection of the two. | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no warm VMs on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | +| `reader` | unset (off) | `{"template": "reader:24.04", "size": "small"}`: serve read-only file calls on a hibernated or archived claim from a disposable reader VM instead of waking it; the named none-lane pool must also be a `pools[]` entry with `warm` of at least 1. See [Reading sleeping claims](#reading-sleeping-claims-without-waking-them). Restart only | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); a lease moves only when a client calls [`/renew`](sandboxd-api.md#post-v1sandboxesidrenew). Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way, unless it claimed `on_expire: archive`, which archives at the lease end whatever this setting. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived; the claim is then gone for good. On S3, chunks and manifests remain until [offline reclamation](#reclaiming-s3-storage). On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. Same node-wide/per-pool split | @@ -166,6 +167,67 @@ fragment the warm pools): | `xlarge` | 4 | 8G | | `2xlarge` | 8 | 16G | +### Reading sleeping claims without waking them + +A client that only lists or reads files (a UI showing an idle agent's +workspace, a tool collecting results) would otherwise wake the claim and +hold its memory until the next idle hibernate, and for an archived claim +fetch the whole checkpoint first. With `reader` set, `fs_stat`, `fs_list`, +`fs_read` and `fs_pull` on a hibernated or archived claim are served from a +reader instead; anything else on the connection wakes the claim as before +([API](sandboxd-api.md#get-v1sandboxesidagent)). + +```json +{ + "reader": {"template": "reader:24.04", "size": "small"}, + "pools": [ + {"template": "reader:24.04", "size": "small", "warm": 2}, + {"template": "rt:24.04", "idle_hibernate_seconds": 600} + ] +} +``` + +The reader pool is an ordinary `pools[]` entry on the none lane, so it is +warmed, refilled and tuned (`PUT /v1/pools`) like any other. Taking a reader +for a claim removes one VM from it until the refill replaces it. A reader is +never shared: it serves one claim, and only the claim's own token reaches it. + +The guest's own kernel does the parsing, as it does for the claim. A reader +is built per claim from the claim's hibernate snapshot (or archive record): + +- the claim's layer blobs, read from the snapshot's sidecar, are attached to + the reader read-only, and its copy-on-write disk as a private copy that the + reader mounts writable, so the guest replays the ext4 journal onto the copy + and never onto the original; +- the reader mounts the layers and the copy, assembles a read-only overlay at + `/view`, and starts the claim's own `silkd` (`/usr/local/bin/silkd`) chrooted + into it on guest vsock port 2051. Paths and symlinks resolve inside the + claim's filesystem, and a write fails at the read-only overlay; +- a hibernated claim's copy is a reflink export of the snapshot, so keep + `data_dir` and cocoon's data directory on a filesystem that supports + reflinks, as for checkpoints; without it the export copies the guest memory + image too. An archived claim fetches only the record's `cow.raw` and + `cocoon.json`: the dir backend copies the file out of the record, the s3 + backend downloads those two objects into the staging cache, which stays until + the claim wakes or is released. + +The reader template must offer `mount`, `chroot`, `env`, `mkdir` and `sync` +and boot a kernel that mounts erofs, ext4 and overlayfs, which the shipped +images do. The claim's image must ship `silkd` at `/usr/local/bin/silkd`, as +the shipped Linux images do; the android image does not, so its claims always +wake. A reader is kept for 30 s after its last connection, then its VM is +destroyed and its copy deleted. A wake never waits for a reader, and a reader +serves only the sleep it was built from: a later hibernate or archive gets a +fresh one. The claim's own code runs in its reader with root, as it did in the +claim; the reader has no NIC, serves one claim, and is destroyed afterwards, so +the claim's bytes never reach a shared VM or the host kernel. + +Whenever a reader cannot be built (no warm reader VM, the disk reserve is low, +a layer blob missing on this node, an image without `silkd` at that path), the +call wakes the claim as it does without the feature. Hibernating a claim then +runs a guest `sync` first, so the snapshot's disk holds what the page cache +still held. Reads show the claim as of that hibernate. + ### Dataset volumes Running claims can change mounted read-only catalog entries through the diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 35a1bdaa..8d7ab390 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -393,7 +393,8 @@ Auth: the sandbox's own token, or the root token by id (operator, like release). Atomically snapshots the VM and stops it, freeing its memory; the next agent access restores it transparently (sessions, processes, and memory state intact — cocoon's hibernate keeps the -snapshot point and the stop coincident). Idempotent on an already-hibernated +snapshot point and the stop coincident), except a read-only file call that a +configured [reader](#get-v1sandboxesidagent) serves without waking it. Idempotent on an already-hibernated sandbox. The TTL keeps running: a hibernated sandbox is still reaped (VM and snapshot) at its deadline, unless it claimed `on_expire: archive` or its pool archives. When to hibernate is the caller's policy — the @@ -964,6 +965,26 @@ while its node's lease is overdue (see claim of a replicating pool that is already open when the lease turns overdue is closed, whether or not it has a replica yet. +A hibernated or archived sandbox wakes on the first frame that opens an RPC, +unless the node configures a [`reader`](deploy.md#reading-sleeping-claims-without-waking-them). +With one, the node answers the `101` before it touches the sandbox, reads the +first frame, and serves `fs_stat`, `fs_list`, `fs_read` and `fs_pull` from a +disposable reader VM that mounts the sandbox's disk read-only: the sandbox stays +hibernated or archived, no `wake` usage event is recorded, and the read stamps +neither the idle nor the archive clock. An `info` frame pipelined ahead of the +call (the SDKs' first frames) rides along to the reader; one alone, with no call +behind it for 100 ms, wakes the sandbox. The first frame on the connection that +is anything else, or any later one, wakes the sandbox and hands the same +connection to its silkd with the unread bytes replayed, so a client sees one +silkd and no close. A connection also moves to the woken sandbox when the sandbox +wakes or sleeps again meanwhile, so reads never come from a stale sleep. Files +and symlinks resolve inside the sandbox's own filesystem as of its last +hibernate. When no reader can be built the call wakes the sandbox as if the +feature were off. Because the `101` has gone out by then, a failure on this path +(unknown sandbox, a refused wake, an unreachable guest) arrives as a silkd +`error` frame (`not_found` for an unknown sandbox, else `internal`) rather than +an HTTP status, and the connection ends. + ## GET /v1/sandboxes/{id}/ports/{port} Auth: the sandbox's own token, or the root `api_token` for a control-plane diff --git a/docs/sdk.md b/docs/sdk.md index e2ffd912..4df80957 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -376,6 +376,11 @@ automatically with the same transparent wake. A claim with a connection live when the sweep checks it (a relay stream, a buffered exec, a preview dial, an egress request) is not swept; the idle clock restarts when that connection ends. +When the node configures a [`reader`](deploy.md#reading-sleeping-claims-without-waking-them), +`Stat`, `ReadFile`, `ReadFileTo`, `ListDir` and `Pull` on a hibernated or archived handle are +served without waking it, as of the last hibernate, and the handle's next call +that is anything else wakes the sandbox on the same connection. + `WithArchiveOnExpire()` makes the lease's end archive the sandbox instead of destroying it, whatever the deployment's archive settings: the node hibernates it if it is running and moves it to the checkpoint store, and the next call diff --git a/docs/silkd.md b/docs/silkd.md index bbe5c54b..68b4cc18 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -6,7 +6,10 @@ the port) for the host only — a peer with any other vsock CID is refused — and started at sysinit in parallel with boot. It is what actually "runs something" inside a sandbox; sandboxd relays SDK connections to it byte-for-byte. Reaching silkd is the -claim-readiness signal — a claim returns only once silkd answers. +claim-readiness signal — a claim returns only once silkd answers. A +[reader VM](deploy.md#reading-sleeping-claims-without-waking-them) runs a +second copy, the claim's own, chrooted into a read-only view of the claim's +filesystem on vsock port 2051. Most users never speak the protocol directly — the [Go SDK](sdk.md) and [Python SDK](sdk-python.md) cover the full surface. This page is the wire From efcc11c2520631fb6b255783a1776ab66ba3f242 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:41:56 +0800 Subject: [PATCH 09/28] pool: reap a reader no connection acquired, and retire the one a registration replaces A reader registered with no idle timer leaked when no connection ever took it (a read racing an exec that wakes the claim), and a second setup overwrote the first, orphaning its VM and disk. Registration now arms the idle timer, retires any reader it replaces, and a setup skips building when a live reader already serves the sleep. --- sandboxd/pool/reader.go | 21 ++++++++++- sandboxd/pool/reader_test.go | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index ac7580c1..ade22057 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -108,7 +108,7 @@ func (m *Manager) acquireReader(ctx context.Context, sb *types.Sandbox) (*reader _, err, _ := rs.flights.Do(sb.ID, func() (any, error) { setupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), readerSetupTimeout) defer cancel() - return nil, m.openReader(setupCtx, sb) + return nil, m.ensureReader(setupCtx, sb) }) if err != nil { return nil, err @@ -149,6 +149,20 @@ func (m *Manager) unregisterReader(r *reader, force bool) bool { return true } +func (m *Manager) ensureReader(ctx context.Context, sb *types.Sandbox) error { + m.mu.Lock() + want := sleepSource(sb) + m.mu.Unlock() + rs := m.readers + rs.mu.Lock() + r := rs.live[sb.ID] + rs.mu.Unlock() + if r != nil && r.source == want { + return nil + } + return m.openReader(ctx, sb) +} + func (m *Manager) openReader(ctx context.Context, sb *types.Sandbox) error { start := time.Now() r := &reader{claim: sb.ID, dir: filepath.Join(m.dataDir, readerDir, sb.ID+"-"+randHex(3))} @@ -169,8 +183,13 @@ func (m *Manager) openReader(ctx context.Context, sb *types.Sandbox) error { } rs := m.readers rs.mu.Lock() + old := rs.live[sb.ID] rs.live[sb.ID] = r + r.idle = time.AfterFunc(readerIdle, func() { m.reapReader(r) }) rs.mu.Unlock() + if old != nil { + m.destroyReader(context.WithoutCancel(ctx), old) + } log.WithFunc("pool.openReader").Infof(ctx, "reader %s for sleeping claim %s ready in %s", r.vm, sb.ID, time.Since(start).Round(time.Millisecond)) return nil } diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 5ec17b6f..c56d44c1 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -148,6 +148,73 @@ func TestReaderIsReleasedAfterItsIdleWindow(t *testing.T) { }) } +func TestReaderNoConnectionAcquiresIsReleasedAfterTheIdleWindow(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + if err := m.ensureReader(t.Context(), sb); err != nil { + t.Fatalf("ensureReader: %v", err) + } + + time.Sleep(readerIdle + time.Second) + synctest.Wait() + if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { + t.Errorf("removes=%v, want the never-acquired reader destroyed", removes) + } + m.readers.mu.Lock() + defer m.readers.mu.Unlock() + if len(m.readers.live) != 0 { + t.Error("the never-acquired reader is still registered") + } + }) +} + +func TestReaderRegistrationRetiresTheOneItReplaces(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + for range 2 { + if err := m.openReader(t.Context(), sb); err != nil { + t.Fatalf("openReader: %v", err) + } + } + removes, setups := readerLifecycle(eng) + if !slices.Contains(removes, "sbx-reader-1") || setups != 2 { + t.Errorf("removes=%v setups=%d, want the first reader destroyed when the second registered", removes, setups) + } + m.readers.mu.Lock() + defer m.readers.mu.Unlock() + if live := m.readers.live[sb.ID]; live == nil || live.vm != "sbx-reader-2" { + t.Errorf("live reader %+v, want the second one", live) + } +} + +func TestReaderEnsureSkipsASetupAnotherFlightFinished(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + for range 2 { + if err := m.ensureReader(t.Context(), sb); err != nil { + t.Fatalf("ensureReader: %v", err) + } + } + if removes, setups := readerLifecycle(eng); setups != 1 || len(removes) != 0 { + t.Errorf("removes=%v setups=%d, want one reader built and none destroyed", removes, setups) + } +} + func TestReaderKeepsTheClaimsLatestSleepOnly(t *testing.T) { eng := newFakeEngine() readerExport(t, eng) From 9526eecdf38fa6c51e222aecc16e8f8bd4dfbb26 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:42:42 +0800 Subject: [PATCH 10/28] server: a read after the claim slept again moves to a fresh reader instead of waking it A connection whose claim was archived or re-hibernated meanwhile handed over and woke the claim, which defeats reading it asleep. It now closes the stale reader and dials one for the new sleep, and hands over only when none can be built. --- docs/sandboxd-api.md | 6 ++-- sandboxd/server/reader.go | 6 ++-- sandboxd/server/reader_test.go | 54 ++++++++++++++++++++++++---------- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 8d7ab390..531c083e 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -976,8 +976,10 @@ call (the SDKs' first frames) rides along to the reader; one alone, with no call behind it for 100 ms, wakes the sandbox. The first frame on the connection that is anything else, or any later one, wakes the sandbox and hands the same connection to its silkd with the unread bytes replayed, so a client sees one -silkd and no close. A connection also moves to the woken sandbox when the sandbox -wakes or sleeps again meanwhile, so reads never come from a stale sleep. Files +silkd and no close. When the sandbox sleeps again meanwhile (a new hibernate or +an archive), the connection's next read moves to a fresh reader of the new sleep; +when it wakes, the connection moves to the woken sandbox, so reads never come +from a stale sleep. Files and symlinks resolve inside the sandbox's own filesystem as of its last hibernate. When no reader can be built the call wakes the sandbox as if the feature were off. Because the `101` has gone out by then, a failure on this path diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index b82f57ea..56fbb586 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -35,7 +35,6 @@ var ( infoOp = wire.Info{}.Op() errReaderGone = errors.New("reader connection lost") - errSleepEnded = errors.New("sandbox is no longer in the sleep the reader serves") errUnreachable = errors.New("guest agent unreachable") ) @@ -143,12 +142,13 @@ func (r *router) run() (net.Conn, func(), bool) { } func (r *router) toReader(head []byte, partial bool) error { + if r.reader != nil && r.s.mgr.SleepingSource(r.id, r.token) != r.source { + r.closeReader() + } if r.reader == nil { if err := r.dialReader(); err != nil { return err } - } else if r.s.mgr.SleepingSource(r.id, r.token) != r.source { - return errSleepEnded } if r.s.mgr.AuditEnabled() { r.audit(head, partial) diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go index fb66a70f..24c77a7a 100644 --- a/sandboxd/server/reader_test.go +++ b/sandboxd/server/reader_test.go @@ -113,23 +113,42 @@ func TestSleepingLaterWriteMovesTheConnectionToTheGuest(t *testing.T) { } } -func TestSleepingReaderStopsServingOnceTheClaimWokeOrSleptAgain(t *testing.T) { - for name, next := range map[string]string{"woke": "", "slept again": "snap-2"} { - t.Run(name, func(t *testing.T) { - env := newSleepingEnv(t) - conn, r := env.upgrade(t) - send(t, conn, statFrame) - expectLine(t, r, "reader") +func TestSleepingReaderStopsServingOnceTheClaimWoke(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + send(t, conn, statFrame) + expectLine(t, r, "reader") - env.reader.setSource(next) - send(t, conn, statFrame) - expectLine(t, r, "guest") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the stale reader connection stayed open") - } - }) + env.reader.setSource("") + send(t, conn, statFrame) + expectLine(t, r, "guest") + select { + case <-env.readerClosed: + case <-time.After(3 * time.Second): + t.Fatal("the stale reader connection stayed open") + } +} + +func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + send(t, conn, statFrame) + expectLine(t, r, "reader") + + env.reader.setSource("snap-2") + send(t, conn, readFrame) + expectLine(t, r, "reader") + select { + case <-env.readerClosed: + case <-time.After(3 * time.Second): + t.Fatal("the stale reader connection stayed open") + } + if wakes, dials := env.reader.counts(); wakes != 0 || dials != 2 { + t.Errorf("wakes=%d reader dials=%d, want a fresh reader for the new sleep and no wake", wakes, dials) + } + want := []string{"reader:" + statFrame, "reader:" + readFrame} + if got := env.frames.all(); !slices.Equal(got, want) { + t.Errorf("frames %q, want %q", got, want) } } @@ -395,6 +414,9 @@ func (f *fakeReader) dial() (net.Conn, string, error) { if f.dialErr != nil { return nil, "", f.dialErr } + if f.src == "" { + return nil, "", pool.ErrNoReader + } f.dials++ relayEnd, readerEnd := net.Pipe() go f.serve(readerEnd) From 1353bcfc1a007aa927fc5878eedff605e3142bdc Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:42:51 +0800 Subject: [PATCH 11/28] docs: a wake waits for a reader's snapshot export --- docs/deploy.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index c52dbc1b..93863c81 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -216,9 +216,10 @@ and boot a kernel that mounts erofs, ext4 and overlayfs, which the shipped images do. The claim's image must ship `silkd` at `/usr/local/bin/silkd`, as the shipped Linux images do; the android image does not, so its claims always wake. A reader is kept for 30 s after its last connection, then its VM is -destroyed and its copy deleted. A wake never waits for a reader, and a reader -serves only the sleep it was built from: a later hibernate or archive gets a -fresh one. The claim's own code runs in its reader with root, as it did in the +destroyed and its copy deleted. A wake waits for a reader only while its +setup exports the hibernate snapshot: milliseconds with a reflink, and the +time to copy the memory image without one. A reader serves only the sleep it +was built from: a later hibernate or archive gets a fresh one. The claim's own code runs in its reader with root, as it did in the claim; the reader has no NIC, serves one claim, and is destroyed afterwards, so the claim's bytes never reach a shared VM or the host kernel. From 218a8871b2ffb1cecf4182c0cd877218d54c6fa6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:43:21 +0800 Subject: [PATCH 12/28] engine: check the claim image's silkd before starting the chrooted one The detached spawn reports started even when the binary is missing, so an image without silkd (android) cost a 10s probe timeout before falling back to a wake. A test -x in the mounted view fails the setup at once. --- docs/deploy.md | 4 +++- sandboxd/engine/reader.go | 3 +++ sandboxd/engine/reader_test.go | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index 93863c81..8265cdb7 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -215,7 +215,9 @@ The reader template must offer `mount`, `chroot`, `env`, `mkdir` and `sync` and boot a kernel that mounts erofs, ext4 and overlayfs, which the shipped images do. The claim's image must ship `silkd` at `/usr/local/bin/silkd`, as the shipped Linux images do; the android image does not, so its claims always -wake. A reader is kept for 30 s after its last connection, then its VM is +wake. A missing silkd is detected right after the view is mounted, so such a +claim pays the snapshot export, one reader VM and the mounts before its call +wakes it, never a probe timeout. A reader is kept for 30 s after its last connection, then its VM is destroyed and its copy deleted. A wake waits for a reader only while its setup exports the hibernate snapshot: milliseconds with a reflink, and the time to copy the memory image without one. A reader serves only the sleep it diff --git a/sandboxd/engine/reader.go b/sandboxd/engine/reader.go index b1aaa8ac..e30b520c 100644 --- a/sandboxd/engine/reader.go +++ b/sandboxd/engine/reader.go @@ -49,6 +49,9 @@ func (e *Engine) StartReader(ctx context.Context, vsockSocket string, d ReaderDi if err := e.silkdExec(ctx, vsockSocket, "mount", "-t", "overlay", "-o", opts, "--", "overlay", readerView); err != nil { return fmt.Errorf("mount view: %w", err) } + if err := e.silkdExec(ctx, vsockSocket, "test", "-x", readerView+readerSilkd); err != nil { + return fmt.Errorf("claim image has no silkd at %s: %w", readerSilkd, err) + } if err := e.silkdSpawn(ctx, vsockSocket, "env", fmt.Sprintf("SILKD_PORT=%d", readerPort), "chroot", readerView, readerSilkd); err != nil { return fmt.Errorf("start view silkd: %w", err) } diff --git a/sandboxd/engine/reader_test.go b/sandboxd/engine/reader_test.go index c4c652be..3fd51ffa 100644 --- a/sandboxd/engine/reader_test.go +++ b/sandboxd/engine/reader_test.go @@ -26,6 +26,7 @@ func TestStartReaderAssemblesTheViewTopLayerFirst(t *testing.T) { {"mount", "-o", "ro", "--", "/dev/vdc", "/run/reader/layer0"}, {"mkdir", "-p", "--", "/view"}, {"mount", "-t", "overlay", "-o", "ro,metacopy=on,redirect_dir=follow,lowerdir=/run/reader/cow/upper:/run/reader/layer1:/run/reader/layer0", "--", "overlay", "/view"}, + {"test", "-x", "/view/usr/local/bin/silkd"}, {"env", "SILKD_PORT=2051", "chroot", "/view", "/usr/local/bin/silkd"}, } fake.mu.Lock() @@ -48,6 +49,25 @@ func TestStartReaderAssemblesTheViewTopLayerFirst(t *testing.T) { } } +func TestStartReaderFailsFastWhenTheClaimImageHasNoSilkd(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureReaderDevices(fake) + fake.mu.Lock() + fake.execFailAt, fake.execCode = 9, 1 + fake.mu.Unlock() + + err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}) + if err == nil || !strings.Contains(err.Error(), "no silkd") { + t.Fatalf("StartReader = %v, want the missing silkd reported", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if last := fake.execCalls[len(fake.execCalls)-1]; last[0] != "test" || slices.Contains(fake.execDetach, true) || slices.Contains(fake.connects, "CONNECT 2051") { + t.Errorf("last command %v, detached %v, connects %v: want the check last, with no spawn and no probe", last, fake.execDetach, fake.connects) + } +} + func TestStartReaderStopsAtTheFirstFailedMount(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) From 3c3d59bfcb23962e3b389f5aa69bd0505d4f54b4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:46:05 +0800 Subject: [PATCH 13/28] review: merge the reader drop paths, share popWarm, inline the pre-hibernate sync, order types in the relay router One dropReader(ctx, r, force) replaces reap/retire/unregister and detaches from the caller's cancellation; popWarm serves the claim and reader paths; the sync is inlined into hibernateLocked; agentSocket stamps activity once after the wake; io.NopCloser replaces noCloser; the router type block leads the file and two comments go. --- sandboxd/pool/claim.go | 5 +-- sandboxd/pool/hibernate.go | 11 ++--- sandboxd/pool/pool.go | 10 +++++ sandboxd/pool/reader.go | 42 +++++------------- sandboxd/pool/reader_test.go | 67 ++++++++++++++++++++++++++--- sandboxd/server/reader.go | 83 +++++++++++++++++------------------- 6 files changed, 127 insertions(+), 91 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 259a40ef..683074ee 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -60,10 +60,7 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, o ClaimOptio var sb *types.Sandbox if p := m.pools[key]; p != nil { p.noteArrival(start) - if n := len(p.warm); n > 0 { - sb = p.warm[n-1] - p.warm = p.warm[:n-1] - } + sb = p.popWarm() } m.mu.Unlock() m.kickRefill() diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index be4d5a30..2a9a21e8 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -43,9 +43,6 @@ func (m *Manager) agentSocket(ctx context.Context, id, token string, readerOK bo if err != nil { return "", nil, nil, err } - if !readerOK { - sb.Touch() - } sb.Hold() sock, err := m.wakeOrRead(ctx, sb, readerOK) switch { @@ -56,9 +53,7 @@ func (m *Manager) agentSocket(ctx context.Context, id, token string, readerOK bo sb.Unhold() return "", nil, nil, err } - if readerOK { - sb.Touch() - } + sb.Touch() return sock, fenced, sb.Unhold, nil } @@ -106,7 +101,9 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return fmt.Errorf("hibernate %s: persist intent: %w", sb.ID, err) } if m.readers != nil { - m.syncForReader(ctx, sb) + if err := m.eng.SyncGuest(ctx, sb.VsockSocket); err != nil { + log.WithFunc("pool.hibernateLocked").Warnf(ctx, "sync %s before hibernate: %v", sb.ID, err) + } } if hibErr := m.eng.Hibernate(ctx, sb.VMName, snap); hibErr != nil { // the engine can report failure after the snapshot landed diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 4de4ae88..c88b4167 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -284,6 +284,16 @@ func (p *pool) applySpec(spec config.PoolSpec) { p.archiveDelete = time.Duration(spec.ArchiveDeleteAfterSeconds) * time.Second } +func (p *pool) popWarm() *types.Sandbox { + n := len(p.warm) + if n == 0 { + return nil + } + sb := p.warm[n-1] + p.warm = p.warm[:n-1] + return sb +} + // trimWarm shrinks p.warm to at most target and returns the trimmed VM names; callers hold m.mu. func (p *pool) trimWarm(target int) []string { var trim []string diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index ade22057..e96fea2e 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -48,7 +48,7 @@ type reader struct { idle *time.Timer } -// DialReader opens a silkd connection that serves a sleeping claim's files read-only from a reader VM, never waking the claim; source names the sleep it serves. +// DialReader opens a silkd connection to a sleeping claim's reader VM without waking the claim. func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Conn, source string, err error) { sb, ok := m.claim(id, token) if !ok { @@ -64,7 +64,7 @@ func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Co conn, err = m.eng.DialReader(ctx, r.sock) if err != nil { m.releaseReader(r) - m.retireReader(r) + m.dropReader(ctx, r, true) return nil, "", fmt.Errorf("%w: %w", ErrNoReader, err) } return &heldConn{Conn: conn, release: sync.OnceFunc(func() { m.releaseReader(r) })}, r.source, nil @@ -103,7 +103,7 @@ func (m *Manager) acquireReader(ctx context.Context, sb *types.Sandbox) (*reader } rs.mu.Unlock() if r != nil { - m.retireReader(r) + m.dropReader(ctx, r, true) } _, err, _ := rs.flights.Do(sb.ID, func() (any, error) { setupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), readerSetupTimeout) @@ -122,31 +122,20 @@ func (m *Manager) releaseReader(r *reader) { defer rs.mu.Unlock() r.conns-- if r.conns == 0 && rs.live[r.claim] == r { - r.idle = time.AfterFunc(readerIdle, func() { m.reapReader(r) }) + r.idle = time.AfterFunc(readerIdle, func() { m.dropReader(context.Background(), r, false) }) } } -func (m *Manager) reapReader(r *reader) { - if m.unregisterReader(r, false) { - m.destroyReader(context.Background(), r) - } -} - -func (m *Manager) retireReader(r *reader) { - if m.unregisterReader(r, true) { - m.destroyReader(context.Background(), r) - } -} - -func (m *Manager) unregisterReader(r *reader, force bool) bool { +func (m *Manager) dropReader(ctx context.Context, r *reader, force bool) { rs := m.readers rs.mu.Lock() - defer rs.mu.Unlock() if rs.live[r.claim] != r || (!force && r.conns > 0) { - return false + rs.mu.Unlock() + return } delete(rs.live, r.claim) - return true + rs.mu.Unlock() + m.destroyReader(context.WithoutCancel(ctx), r) } func (m *Manager) ensureReader(ctx context.Context, sb *types.Sandbox) error { @@ -185,7 +174,7 @@ func (m *Manager) openReader(ctx context.Context, sb *types.Sandbox) error { rs.mu.Lock() old := rs.live[sb.ID] rs.live[sb.ID] = r - r.idle = time.AfterFunc(readerIdle, func() { m.reapReader(r) }) + r.idle = time.AfterFunc(readerIdle, func() { m.dropReader(context.Background(), r, false) }) rs.mu.Unlock() if old != nil { m.destroyReader(context.WithoutCancel(ctx), old) @@ -256,10 +245,7 @@ func (m *Manager) takeReaderVM() (*types.Sandbox, error) { m.mu.Lock() var sb *types.Sandbox if p, ok := m.activePool(m.readers.key); ok { - if n := len(p.warm); n > 0 { - sb = p.warm[n-1] - p.warm = p.warm[:n-1] - } + sb = p.popWarm() } m.mu.Unlock() m.kickRefill() @@ -291,12 +277,6 @@ func (m *Manager) destroyReader(ctx context.Context, r *reader) { } } -func (m *Manager) syncForReader(ctx context.Context, sb *types.Sandbox) { - if err := m.eng.SyncGuest(ctx, sb.VsockSocket); err != nil { - log.WithFunc("pool.syncForReader").Warnf(ctx, "sync %s before hibernate: %v", sb.ID, err) - } -} - // sleepSource names a settled sleep's snapshot or record; callers hold m.mu. func sleepSource(sb *types.Sandbox) string { if sb.Failed != "" || sb.PendingSnap != "" { diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index c56d44c1..cfaab5fe 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -30,7 +30,7 @@ func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) - if _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { t.Fatalf("WakeAgentSocketOrRead = %v, want ErrSleeping", err) } if sb.Busy() { @@ -227,7 +227,7 @@ func TestReaderKeepsTheClaimsLatestSleepOnly(t *testing.T) { t.Fatalf("DialReader: %v", err) } _ = first.Close() - if _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } if got := m.SleepingSource(sb.ID, sb.Token); got != "" { @@ -265,7 +265,7 @@ func TestReaderServesAnArchivedClaimFromTwoFilesOfItsRecord(t *testing.T) { mustArchive(t, m, sb) clones := len(eng.clones) + len(eng.colds) - if _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { t.Fatalf("WakeAgentSocketOrRead of an archived claim = %v, want ErrSleeping", err) } conn, source, err := m.DialReader(t.Context(), sb.ID, sb.Token) @@ -440,6 +440,63 @@ func TestReaderThatStopsAnsweringIsRetired(t *testing.T) { } } +func TestWakeAgentSocketStampsActivityOnlyWhenItServesTheGuest(t *testing.T) { + for _, on := range []bool{false, true} { + t.Run(fmt.Sprintf("reader=%t", on), func(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + if on { + m = newReaderManager(t, eng) + } + sb := mustClaim(t, m, testKey) + backdate(m, sb, time.Hour) + _, _, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("WakeAgentSocketOrRead: %v", err) + } + defer done() + if age := time.Since(sb.LastSeen()); age > time.Minute { + t.Errorf("a served connection left the claim idle for %v", age) + } + hibernateClaim(t, m, sb) + backdate(m, sb, time.Hour) + if _, _, _, err = m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); on != errors.Is(err, ErrSleeping) { + t.Fatalf("WakeAgentSocketOrRead of a hibernated claim = %v with the reader %t", err, on) + } + if age := time.Since(sb.LastSeen()); on && age < 30*time.Minute { + t.Errorf("a deferred read stamped the claim's idle clock (idle %v)", age) + } + }) + } +} + +func TestReaderOfAStaleSleepIsRetiredEvenWithAConnectionOpen(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + open, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + defer func() { _ = open.Close() }() + if _, _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("wake: %v", err) + } + hibernateClaim(t, m, sb) + + next, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader after the second sleep: %v", err) + } + _ = next.Close() + if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { + t.Errorf("removes=%v, want the stale reader destroyed although its connection is open", removes) + } +} + func TestReaderOffWakesAsBefore(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -449,7 +506,7 @@ func TestReaderOffWakesAsBefore(t *testing.T) { if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { t.Errorf("DialReader with the reader off = %v, want ErrNoReader", err) } - sock, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + sock, _, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) if err != nil || sock == "" { t.Fatalf("WakeAgentSocketOrRead with the reader off = %q, %v, want a wake", sock, err) } @@ -464,7 +521,7 @@ func TestReaderOnLeavesRunningClaimsAlone(t *testing.T) { m := newReaderManager(t, eng) sb := mustClaim(t, m, testKey) - sock, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + sock, _, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) if err != nil || sock != sb.VsockSocket { t.Fatalf("WakeAgentSocketOrRead of a running claim = %q, %v, want its own socket", sock, err) } diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index 56fbb586..ae948bc9 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -38,46 +38,6 @@ var ( errUnreachable = errors.New("guest agent unreachable") ) -// serveSleeping answers the 101 before reading any frame, since a client's first frame can wait on it. -func (s *Server) serveSleeping(ctx context.Context, w http.ResponseWriter, id, token string) { - client, clientBuf, ok := hijack(ctx, w) - if !ok { - return - } - release, ok := s.trackRelay(client, noCloser{}) - if !ok { - return - } - defer release() - _ = client.SetDeadline(time.Time{}) - keepAlive(client) - if _, err := client.Write(switchingProtocols); err != nil { - return - } - rt := &router{s: s, ctx: ctx, id: id, token: token, client: client, br: bufio.NewReaderSize(clientReader(client, clientBuf), routeBuf)} - guest, done, ok := rt.run() - if !ok { - return - } - defer done() - defer func() { _ = guest.Close() }() - s.retargetRelay(client, guest) - copyBoth(client, s.audited(ctx, id, io.MultiReader(bytes.NewReader(rt.replay), rt.br)), guest, silkdEnds) -} - -func (s *Server) openGuest(ctx context.Context, id, token string) (net.Conn, func(), error) { - sock, _, done, err := s.mgr.WakeAgentSocket(ctx, id, token) - if err != nil { - return nil, nil, err - } - guest, err := s.dialer.DialSilkd(ctx, sock) - if err != nil { - done() - return nil, nil, fmt.Errorf("%w: %w", errUnreachable, err) - } - return guest, done, nil -} - type router struct { s *Server ctx context.Context @@ -93,8 +53,7 @@ type router struct { superseded atomic.Bool pending []byte - // replay is what the guest has not seen when the router hands over. - replay []byte + replay []byte } func (r *router) run() (net.Conn, func(), bool) { @@ -286,6 +245,42 @@ func (r *router) audit(head []byte, partial bool) { r.s.mgr.Audit(r.ctx, r.id, bytes.TrimSuffix(head, newline)) } -type noCloser struct{} +// serveSleeping answers the 101 before reading any frame, since a client's first frame can wait on it. +func (s *Server) serveSleeping(ctx context.Context, w http.ResponseWriter, id, token string) { + client, clientBuf, ok := hijack(ctx, w) + if !ok { + return + } + release, ok := s.trackRelay(client, io.NopCloser(nil)) + if !ok { + return + } + defer release() + _ = client.SetDeadline(time.Time{}) + keepAlive(client) + if _, err := client.Write(switchingProtocols); err != nil { + return + } + rt := &router{s: s, ctx: ctx, id: id, token: token, client: client, br: bufio.NewReaderSize(clientReader(client, clientBuf), routeBuf)} + guest, done, ok := rt.run() + if !ok { + return + } + defer done() + defer func() { _ = guest.Close() }() + s.retargetRelay(client, guest) + copyBoth(client, s.audited(ctx, id, io.MultiReader(bytes.NewReader(rt.replay), rt.br)), guest, silkdEnds) +} -func (noCloser) Close() error { return nil } +func (s *Server) openGuest(ctx context.Context, id, token string) (net.Conn, func(), error) { + sock, _, done, err := s.mgr.WakeAgentSocket(ctx, id, token) + if err != nil { + return nil, nil, err + } + guest, err := s.dialer.DialSilkd(ctx, sock) + if err != nil { + done() + return nil, nil, fmt.Errorf("%w: %w", errUnreachable, err) + } + return guest, done, nil +} From 65d8e861c91a999265002cb7384d8fa5a4735027 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:58:56 +0800 Subject: [PATCH 14/28] pool: a reader serves only a sleep whose pre-hibernate sync succeeded A write the guest makes after the sync returns stays in memory and misses the disk copy, and a failed sync was only logged. Hibernate now records ReaderSafe (persisted with the claim, cleared on wake and on adopting a pending hibernate, absent on sleeps from before the feature), and reads of an unsafe sleep wake the claim as without the feature. The docs state that a reader shows the disk as of the sync right before the pause. --- docs/deploy.md | 9 ++++-- docs/sandboxd-api.md | 7 +++-- docs/sdk.md | 2 +- sandboxd/pool/archive.go | 6 ++-- sandboxd/pool/claims.go | 3 +- sandboxd/pool/hibernate.go | 15 ++++++--- sandboxd/pool/pool_test.go | 3 +- sandboxd/pool/reader.go | 4 +-- sandboxd/pool/reader_test.go | 60 ++++++++++++++++++++++++++++++++++++ sandboxd/pool/reconcile.go | 2 +- sandboxd/types/types.go | 1 + 11 files changed, 94 insertions(+), 18 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 8265cdb7..bca27da4 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -227,9 +227,12 @@ the claim's bytes never reach a shared VM or the host kernel. Whenever a reader cannot be built (no warm reader VM, the disk reserve is low, a layer blob missing on this node, an image without `silkd` at that path), the -call wakes the claim as it does without the feature. Hibernating a claim then -runs a guest `sync` first, so the snapshot's disk holds what the page cache -still held. Reads show the claim as of that hibernate. +call wakes the claim as it does without the feature. With the feature on, +every hibernate runs a guest `sync` right before the pause, and a reader shows +the disk as of that `sync`: a write the guest makes in the instant between the +`sync` and the pause is visible only after a wake. A hibernate whose `sync` +fails or times out is marked not reader-safe, as is any sleep that began before +the feature was on, and its reads wake the claim as without the feature. ### Dataset volumes diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 531c083e..f23f1bce 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -980,8 +980,11 @@ silkd and no close. When the sandbox sleeps again meanwhile (a new hibernate or an archive), the connection's next read moves to a fresh reader of the new sleep; when it wakes, the connection moves to the woken sandbox, so reads never come from a stale sleep. Files -and symlinks resolve inside the sandbox's own filesystem as of its last -hibernate. When no reader can be built the call wakes the sandbox as if the +and symlinks resolve inside the sandbox's own filesystem as of the guest `sync` +that runs right before the last hibernate's pause; a write the guest made in +the instant between that `sync` and the pause is visible only after a wake. A +hibernate whose `sync` failed, or one made before the reader was configured, +is not served by a reader: its reads wake the sandbox. When no reader can be built the call wakes the sandbox as if the feature were off. Because the `101` has gone out by then, a failure on this path (unknown sandbox, a refused wake, an unreachable guest) arrives as a silkd `error` frame (`not_found` for an unknown sandbox, else `internal`) rather than diff --git a/docs/sdk.md b/docs/sdk.md index 4df80957..2970bcb0 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -378,7 +378,7 @@ egress request) is not swept; the idle clock restarts when that connection ends. When the node configures a [`reader`](deploy.md#reading-sleeping-claims-without-waking-them), `Stat`, `ReadFile`, `ReadFileTo`, `ListDir` and `Pull` on a hibernated or archived handle are -served without waking it, as of the last hibernate, and the handle's next call +served without waking it, as of the guest `sync` just before the last hibernate, and the handle's next call that is anything else wakes the sandbox on the same connection. `WithArchiveOnExpire()` makes the lease's end archive the sandbox instead of diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 03062c69..af26aa17 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -258,8 +258,8 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc m.mu.Unlock() return false, false } - ck, deadline, epoch := sb.ArchiveCk, sb.Deadline, sb.ArchiveEpoch - sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch = vmName, sock, "", 0 + ck, deadline, epoch, safe := sb.ArchiveCk, sb.Deadline, sb.ArchiveEpoch, sb.ReaderSafe + sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch, sb.ReaderSafe = vmName, sock, "", 0, false sb.Deadline = time.Now().Add(clampTTL(types.Seconds(sb.LeaseSeconds))) // a woken sandbox is a fresh lease js := m.store.set(sb) m.mu.Unlock() @@ -267,7 +267,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc m.mu.Lock() var rb claimSnapshot if m.claimed[sb.ID] == sb { - sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch, sb.Deadline = "", "", ck, epoch, deadline + sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch, sb.Deadline, sb.ReaderSafe = "", "", ck, epoch, deadline, safe rb = m.store.set(sb) } m.mu.Unlock() diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index f7712913..12bcbeb3 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -54,6 +54,7 @@ type claimDTO struct { TAP string `json:"tap,omitempty"` HibernateSnap string `json:"hibernate_snap,omitempty"` PendingSnap string `json:"pending_snap,omitempty"` + ReaderSafe bool `json:"reader_safe,omitempty"` ArchiveCk string `json:"archive_ck,omitempty"` ArchiveEpoch int64 `json:"archive_epoch,omitzero"` ArchivePending string `json:"archive_pending,omitempty"` @@ -265,7 +266,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket, - TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, + TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ReaderSafe: sb.ReaderSafe, ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, ReplicaPending: sb.ReplicaPending, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 2a9a21e8..d192b43c 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -100,11 +100,17 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error m.mu.Unlock() return fmt.Errorf("hibernate %s: persist intent: %w", sb.ID, err) } + readerSafe := false if m.readers != nil { - if err := m.eng.SyncGuest(ctx, sb.VsockSocket); err != nil { - log.WithFunc("pool.hibernateLocked").Warnf(ctx, "sync %s before hibernate: %v", sb.ID, err) + err := m.eng.SyncGuest(ctx, sb.VsockSocket) + if err != nil { + log.WithFunc("pool.hibernateLocked").Warnf(ctx, "sync %s before hibernate, reads will wake it: %v", sb.ID, err) } + readerSafe = err == nil } + m.mu.Lock() + sb.ReaderSafe = readerSafe + m.mu.Unlock() if hibErr := m.eng.Hibernate(ctx, sb.VMName, snap); hibErr != nil { // the engine can report failure after the snapshot landed adopted, resolveErr := m.settlePendingSnap(ctx, sb) @@ -145,7 +151,7 @@ func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bo sb.Transition.Lock() defer sb.Transition.Unlock() if sb.ArchiveCk != "" { - if readerOK { + if readerOK && sb.ReaderSafe { return "", ErrSleeping } return m.wakeArchived(ctx, sb) @@ -168,7 +174,7 @@ func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bo if sb.Key.Net == types.NetEgress { return "", fmt.Errorf("wake %s: egress lane cannot resume from hibernation", sb.ID) } - if readerOK { + if readerOK && sb.ReaderSafe { return "", ErrSleeping } if err := m.diskErr(); err != nil { @@ -292,6 +298,7 @@ func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, sb.HibernateSnap = snap sb.VsockSocket = sock sb.PendingSnap = "" + sb.ReaderSafe = sb.ReaderSafe && snap != "" if now := time.Now(); snap == "" && sb.OnExpire == types.ExpireArchive && !m.tenantGone(sb.Tenant) && now.After(sb.Deadline) { sb.Deadline = now.Add(clampTTL(types.Seconds(sb.LeaseSeconds))) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index e44ca5b2..54f55143 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1225,6 +1225,7 @@ type fakeEngine struct { readerDials []string startReaderErr error readerDialErr error + syncErr error caInstalls []string warmups [][]string warmupSocks []string @@ -1683,7 +1684,7 @@ func (f *fakeEngine) SyncGuest(_ context.Context, vsockSocket string) error { defer f.mu.Unlock() f.syncs = append(f.syncs, vsockSocket) f.volumeOps = append(f.volumeOps, "sync") - return nil + return f.syncErr } func (f *fakeEngine) StartReader(_ context.Context, vsockSocket string, d engine.ReaderDisks) error { diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index e96fea2e..921d68d1 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -201,7 +201,7 @@ func (m *Manager) readerSource(ctx context.Context, sb *types.Sandbox, r *reader func (m *Manager) captureSleeping(ctx context.Context, sb *types.Sandbox, r *reader) (archived bool, err error) { sb.Transition.Lock() defer sb.Transition.Unlock() - if sb.HibernateSnap == "" && sb.ArchiveCk == "" { + if (sb.HibernateSnap == "" && sb.ArchiveCk == "") || !sb.ReaderSafe { return false, errNotSleeping } if sb.HibernateSnap == "" { @@ -279,7 +279,7 @@ func (m *Manager) destroyReader(ctx context.Context, r *reader) { // sleepSource names a settled sleep's snapshot or record; callers hold m.mu. func sleepSource(sb *types.Sandbox) string { - if sb.Failed != "" || sb.PendingSnap != "" { + if sb.Failed != "" || sb.PendingSnap != "" || !sb.ReaderSafe { return "" } return cmp.Or(sb.HibernateSnap, sb.ArchiveCk) diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index cfaab5fe..dfbefa06 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -497,6 +497,66 @@ func TestReaderOfAStaleSleepIsRetiredEvenWithAConnectionOpen(t *testing.T) { } } +func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { + for _, tt := range []struct { + name string + syncErr error + safe bool + }{ + {"sync ok", nil, true}, + {"sync fails", errors.New("guest unreachable"), false}, + } { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + readerExport(t, eng) + eng.syncErr = tt.syncErr + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + if got, _ := newClaimStore(m.dataDir, false).load(); got[sb.ID] == nil || got[sb.ID].ReaderSafe != tt.safe { + t.Errorf("persisted reader_safe = %+v, want %t", got[sb.ID], tt.safe) + } + if got := m.SleepingSource(sb.ID, sb.Token); (got != "") != tt.safe { + t.Errorf("SleepingSource = %q with a safe sleep %t", got, tt.safe) + } + if tt.safe { + if _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + t.Fatalf("a safe sleep answered %v, want ErrSleeping", err) + } + return + } + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Errorf("DialReader of an unsafe sleep = %v, want ErrNoReader", err) + } + sock, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + if err != nil || sock == "" { + t.Fatalf("a read of an unsafe sleep = %q, %v, want a wake", sock, err) + } + defer done() + if len(eng.restores) != 1 || len(eng.startedReaders) != 0 { + t.Errorf("restores=%v reader setups=%d, want the claim woken and no reader", eng.restores, len(eng.startedReaders)) + } + }) + } +} + +func TestWakeClearsReaderSafe(t *testing.T) { + eng := newFakeEngine() + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + if _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("wake: %v", err) + } else { + done() + } + if got, _ := newClaimStore(m.dataDir, false).load(); got[sb.ID] == nil || got[sb.ID].ReaderSafe { + t.Errorf("persisted claim after a wake = %+v, want reader_safe cleared", got[sb.ID]) + } +} + func TestReaderOffWakesAsBefore(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 1cf4ba80..3bfa3cf1 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -57,7 +57,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { sb.PendingSnap = "" case ok && sb.PendingSnap != "" && (snapsErr != nil || slices.Contains(snaps, sb.PendingSnap)): // the journaled intent names the wake image, so adopt it when the snapshot is there - sb.HibernateSnap, sb.PendingSnap = sb.PendingSnap, "" + sb.HibernateSnap, sb.PendingSnap, sb.ReaderSafe = sb.PendingSnap, "", false adopted = append(adopted, sb) case ok: // the VMM died with the host or on its own; the liveness pass restarts or fails it diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 7d7b8066..a7a9983b 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -571,6 +571,7 @@ type Sandbox struct { HibernateSnap string `json:"hibernate_snap,omitempty"` // PendingSnap is the journaled intent of a hibernate in flight, cleared by its commit. PendingSnap string `json:"pending_snap,omitempty"` + ReaderSafe bool `json:"reader_safe,omitempty"` // ArchiveCk names the store checkpoint holding an archived sandbox; empty means local. ArchiveCk string `json:"archive_ck,omitempty"` // ArchiveEpoch is the archive registry epoch this node last held for the claim; zero means no row. From cb51e2b66f4780e88b06944fab7341b7c4095c73 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:00:49 +0800 Subject: [PATCH 15/28] store: FetchFile writes into the caller's file, so a reader's fetch leaves nothing in the shared cache A FetchFile racing a release could recreate cache/ in the s3 staging and leave a file nothing swept until restart. FetchFile now takes a destination: the reader fetches into its own directory, which idle release, retire and boot reconcile already delete. The dir backend clones the record's file there. --- docs/deploy.md | 7 ++-- sandboxd/pool/reader.go | 18 +++----- sandboxd/pool/reader_test.go | 18 +++++--- sandboxd/store/dir/dir.go | 15 +++---- sandboxd/store/s3/s3.go | 59 ++++++++------------------- sandboxd/store/s3/s3_test.go | 56 +++++++++++++++++-------- sandboxd/store/store.go | 4 +- sandboxd/store/storetest/storetest.go | 16 +++++--- 8 files changed, 99 insertions(+), 94 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index bca27da4..c32ffc10 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -207,9 +207,10 @@ is built per claim from the claim's hibernate snapshot (or archive record): `data_dir` and cocoon's data directory on a filesystem that supports reflinks, as for checkpoints; without it the export copies the guest memory image too. An archived claim fetches only the record's `cow.raw` and - `cocoon.json`: the dir backend copies the file out of the record, the s3 - backend downloads those two objects into the staging cache, which stays until - the claim wakes or is released. + `cocoon.json` straight into the reader's own directory: the dir backend + copies the file out of the record, the s3 backend downloads those two + objects. Nothing lands in the store's shared fetch cache, so releasing the + reader removes every byte it fetched. The reader template must offer `mount`, `chroot`, `env`, `mkdir` and `sync` and boot a kernel that mounts erofs, ext4 and overlayfs, which the shipped diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index 921d68d1..7062fbad 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -224,21 +224,15 @@ func (m *Manager) captureSleeping(ctx context.Context, sb *types.Sandbox, r *rea } func (m *Manager) fetchReaderDisk(ctx context.Context, r *reader) (string, error) { - disk, err := m.ckpts.FetchFile(ctx, r.source, readerCowFile) - if err != nil { - return "", archiveFetchErr(err) - } - sidecar, err := m.ckpts.FetchFile(ctx, r.source, readerSidecarFile) - if err != nil { - return "", archiveFetchErr(err) - } - if err = os.MkdirAll(r.dir, 0o750); err != nil { + if err := os.MkdirAll(r.dir, 0o750); err != nil { return "", err } - if err = utils.CloneFile(disk, filepath.Join(r.dir, readerCowFile)); err != nil { - return "", fmt.Errorf("copy archive disk: %w", err) + for _, name := range []string{readerCowFile, readerSidecarFile} { + if err := m.ckpts.FetchFile(ctx, r.source, name, filepath.Join(r.dir, name)); err != nil { + return "", archiveFetchErr(err) + } } - return sidecar, nil + return filepath.Join(r.dir, readerSidecarFile), nil } func (m *Manager) takeReaderVM() (*types.Sandbox, error) { diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index dfbefa06..12f31045 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -299,6 +299,14 @@ func TestReaderServesAnArchivedClaimFromTwoFilesOfItsRecord(t *testing.T) { if !strings.HasPrefix(cow.Path, filepath.Join(m.dataDir, readerDir)) { t.Errorf("reader attached %s, want a private copy under %s", cow.Path, readerDir) } + + m.readers.mu.Lock() + live := m.readers.live[sb.ID] + m.readers.mu.Unlock() + m.dropReader(t.Context(), live, true) + if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { + t.Errorf("a released archived reader left %d entries under %s", len(entries), readerDir) + } } func TestReaderRefusesWhatAWakeRefuses(t *testing.T) { @@ -522,7 +530,7 @@ func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { t.Errorf("SleepingSource = %q with a safe sleep %t", got, tt.safe) } if tt.safe { - if _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { t.Fatalf("a safe sleep answered %v, want ErrSleeping", err) } return @@ -530,7 +538,7 @@ func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { t.Errorf("DialReader of an unsafe sleep = %v, want ErrNoReader", err) } - sock, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + sock, _, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) if err != nil || sock == "" { t.Fatalf("a read of an unsafe sleep = %q, %v, want a wake", sock, err) } @@ -547,7 +555,7 @@ func TestWakeClearsReaderSafe(t *testing.T) { m := newReaderManager(t, eng) sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) - if _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } else { done() @@ -714,11 +722,11 @@ func (s *fetchCountingStore) Fetch(ctx context.Context, id string) (string, []by return s.Store.Fetch(ctx, id) } -func (s *fetchCountingStore) FetchFile(ctx context.Context, id, rel string) (string, error) { +func (s *fetchCountingStore) FetchFile(ctx context.Context, id, rel, dst string) error { s.mu.Lock() s.fetched = append(s.fetched, rel) s.mu.Unlock() - return s.Store.FetchFile(ctx, id, rel) + return s.Store.FetchFile(ctx, id, rel, dst) } func (s *fetchCountingStore) files() []string { diff --git a/sandboxd/store/dir/dir.go b/sandboxd/store/dir/dir.go index 5eccef38..92ca8ccc 100644 --- a/sandboxd/store/dir/dir.go +++ b/sandboxd/store/dir/dir.go @@ -90,19 +90,16 @@ func (d *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e return dir, meta, digest, nil } -func (d *Store) FetchFile(ctx context.Context, id, rel string) (string, error) { +func (d *Store) FetchFile(ctx context.Context, id, rel, dst string) error { meta, err := d.ReadMeta(ctx, id) if err != nil { - return "", err + return err } - path := filepath.Join(d.root, id, store.ExportGen(meta), rel) - switch _, statErr := os.Stat(path); { - case errors.Is(statErr, fs.ErrNotExist): - return "", store.ErrNotFound - case statErr != nil: - return "", statErr + err = utils.CloneFile(filepath.Join(d.root, id, store.ExportGen(meta), rel), dst) + if errors.Is(err, fs.ErrNotExist) { + return store.ErrNotFound } - return path, nil + return err } func (d *Store) ReadMeta(_ context.Context, id string) ([]byte, error) { diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 5f49d3bc..51206e9f 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -154,29 +154,29 @@ func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e return export, meta, digest, nil } -func (s *Store) FetchFile(ctx context.Context, id, rel string) (string, error) { +func (s *Store) FetchFile(ctx context.Context, id, rel, dst string) error { + ctx, cancel := context.WithTimeout(ctx, transferBudget) + defer cancel() meta, _, err := s.readMeta(ctx, id) if err != nil { - return "", err + return err } - path := filepath.Join(s.staging, "cache", id, store.ExportGenHash(meta)+"-"+strings.ReplaceAll(rel, "/", "_")) - if _, statErr := os.Stat(path); statErr == nil { - return path, nil + man, err := s.readManifest(ctx, id, store.ExportGen(meta)) + if err != nil { + return err } - flight := s.fetches.DoChan(path, func() (any, error) { - fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), transferBudget) - defer cancel() - return nil, s.populateFile(fctx, id, store.ExportGen(meta), rel, path) - }) - select { - case res := <-flight: - if res.Err != nil { - return "", res.Err - } - case <-ctx.Done(): - return "", ctx.Err() + i := slices.IndexFunc(man.Files, func(f manifestFile) bool { return f.Path == rel }) + if i < 0 { + return store.ErrNotFound } - return path, nil + part := dst + ".part" + defer func() { _ = os.Remove(part) }() + file := man.Files[i] + file.Path = filepath.Base(part) + if err = s.materialize(ctx, manifest{Lineage: man.Lineage, Files: []manifestFile{file}}, filepath.Dir(part)); err != nil { + return err + } + return os.Rename(part, dst) } func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { @@ -439,29 +439,6 @@ func (s *Store) materialize(ctx context.Context, man manifest, root string) (err return g.Wait() } -// populateFile materializes one manifest file through the chunk path of a full fetch, so holes stay holes. -func (s *Store) populateFile(ctx context.Context, id, gen, rel, path string) error { - if _, err := os.Stat(path); err == nil { - return nil - } - man, err := s.readManifest(ctx, id, gen) - if err != nil { - return err - } - i := slices.IndexFunc(man.Files, func(f manifestFile) bool { return f.Path == rel }) - if i < 0 { - return store.ErrNotFound - } - part := path + ".part" - defer func() { _ = os.Remove(part) }() - one := manifest{Lineage: man.Lineage, Files: []manifestFile{man.Files[i]}} - one.Files[0].Path = filepath.Base(part) - if err = s.materialize(ctx, one, filepath.Dir(part)); err != nil { - return err - } - return os.Rename(part, path) -} - func (s *Store) fetchChunk(ctx context.Context, key string, f *os.File, size int64, chunk manifestChunk) error { out, err := s.get(ctx, key) if errors.Is(err, store.ErrNotFound) { diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index 4de71b46..881175c7 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -732,11 +732,11 @@ func TestFetchFileDownloadsOnlyTheRequestedFile(t *testing.T) { } fake.resetRequests() - path, err := st.FetchFile(t.Context(), id, "disk.img") - if err != nil { + dst := filepath.Join(t.TempDir(), "disk.img") + if err := st.FetchFile(t.Context(), id, "disk.img", dst); err != nil { t.Fatalf("FetchFile: %v", err) } - if got, readErr := os.ReadFile(path); readErr != nil || string(got) != "disk-bytes" { + if got, readErr := os.ReadFile(dst); readErr != nil || string(got) != "disk-bytes" { t.Fatalf("fetched file: %q, %v", got, readErr) } memory := sha256.Sum256([]byte("memory-bytes")) @@ -744,13 +744,8 @@ func TestFetchFileDownloadsOnlyTheRequestedFile(t *testing.T) { if n := fake.requestCount(http.MethodGet, memoryKey); n != 0 { t.Errorf("GET %s = %d requests, want 0", memoryKey, n) } - disk := sha256.Sum256([]byte("disk-bytes")) - diskKey := "ck/chunks/" + id + "/" + hex.EncodeToString(disk[:]) - if again, err := st.FetchFile(t.Context(), id, "disk.img"); err != nil || again != path { - t.Fatalf("second FetchFile = %q, %v, want the cached %q", again, err, path) - } - if n := fake.requestCount(http.MethodGet, diskKey); n != 1 { - t.Errorf("GET %s = %d requests, want 1 across two fetches", diskKey, n) + if _, err := os.Stat(filepath.Join(st.staging, "cache")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("FetchFile touched the shared fetch cache: %v", err) } } @@ -760,23 +755,50 @@ func TestFetchFileInstallsOnlyACompleteDownload(t *testing.T) { fake := &fakeS3{objects: map[string][]byte{}, exportGate: make(chan struct{}), exportEntered: make(chan struct{}, 1)} st := newTestStore(t, fake) publishRecord(t, st, id, meta, "disk-bytes") - cached := filepath.Join(st.staging, "cache", id, store.ExportGenHash(meta)+"-disk.img") + dst := filepath.Join(t.TempDir(), "disk.img") fetched := make(chan error, 1) go func() { - _, err := st.FetchFile(t.Context(), id, "disk.img") - fetched <- err + fetched <- st.FetchFile(t.Context(), id, "disk.img", dst) }() <-fake.exportEntered - if _, err := os.Stat(cached); !errors.Is(err, os.ErrNotExist) { - t.Errorf("a download in flight is already visible at the cache path: %v", err) + if _, err := os.Stat(dst); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a download in flight is already visible at the destination: %v", err) } close(fake.exportGate) if err := <-fetched; err != nil { t.Fatalf("FetchFile: %v", err) } - if got, err := os.ReadFile(cached); err != nil || string(got) != "disk-bytes" { - t.Errorf("cached file %q, %v, want the object's bytes", got, err) + if got, err := os.ReadFile(dst); err != nil || string(got) != "disk-bytes" { + t.Errorf("destination %q, %v, want the object's bytes", got, err) + } + if _, err := os.Stat(dst + ".part"); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the partial download survived: %v", err) + } +} + +func TestFetchFileKeepsHoles(t *testing.T) { + const id = "ck_00000000000000a2" + st := newTestStore(t, &fakeS3{objects: map[string][]byte{}}) + size := 12*store.DigestChunkSize + 4096 + staging, src := stageSparseRecord(t, st, id, []byte(`{"id":"`+id+`"}`), size, map[int64]string{0: "head", 11*store.DigestChunkSize + 10: "tail"}) + srcSum := fileSum(t, src) + if err := st.Publish(t.Context(), staging, id, ""); err != nil { + t.Fatalf("Publish: %v", err) + } + + dst := filepath.Join(t.TempDir(), "disk.img") + if err := st.FetchFile(t.Context(), id, "disk.img", dst); err != nil { + t.Fatalf("FetchFile: %v", err) + } + if info, err := os.Stat(dst); err != nil || info.Size() != size { + t.Fatalf("fetched size = %v, %v, want %d", info, err, size) + } + if fileSum(t, dst) != srcSum { + t.Errorf("fetched content differs from the source") + } + if blocks, limit := allocatedBytes(t, dst), 2*store.DigestChunkSize+1<<20; blocks > limit { + t.Errorf("fetched file allocates %d bytes, want at most %d for two data chunks", blocks, limit) } } diff --git a/sandboxd/store/store.go b/sandboxd/store/store.go index 62f43872..0cc61a09 100644 --- a/sandboxd/store/store.go +++ b/sandboxd/store/store.go @@ -65,8 +65,8 @@ type Store interface { PublishDigested(ctx context.Context, staging, id, lineage string) (string, error) // Fetch materializes a record's export locally; the caller holds the record lock until the clone ends. Fetch(ctx context.Context, id string) (dir string, meta []byte, digest string, err error) - // FetchFile materializes one file of a record's export locally, read-only to the caller, without fetching the rest of the record. - FetchFile(ctx context.Context, id, rel string) (string, error) + // FetchFile writes one file of a record's export to dst, without fetching the rest of the record; dst is the caller's to delete. + FetchFile(ctx context.Context, id, rel, dst string) error // ReadMeta returns a record's metadata, or an error when the record does not exist. ReadMeta(ctx context.Context, id string) ([]byte, error) // Metas lists the metadata and digest of every record in this instance's id namespace. diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index 4595a070..60a05f01 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -77,21 +77,27 @@ func RunContract(t *testing.T, st store.Store) { t.Errorf("second-generation export: %q, %v", got, readErr) } - filePath, err := st.FetchFile(ctx, id, "disk2.img") - if err != nil { + dst := filepath.Join(t.TempDir(), "fetched") + if err = st.FetchFile(ctx, id, "disk2.img", dst); err != nil { t.Fatalf("FetchFile: %v", err) } - if got, readErr := os.ReadFile(filePath); readErr != nil || string(got) != "second-gen" { //nolint:gosec // test path + if got, readErr := os.ReadFile(dst); readErr != nil || string(got) != "second-gen" { //nolint:gosec // test path t.Errorf("fetched file: %q, %v", got, readErr) } - if _, err = st.FetchFile(ctx, id, "absent.img"); !errors.Is(err, store.ErrNotFound) { + missing := filepath.Join(t.TempDir(), "missing") + if err = st.FetchFile(ctx, id, "absent.img", missing); !errors.Is(err, store.ErrNotFound) { t.Errorf("FetchFile of an absent file: %v, want store.ErrNotFound", err) } + for _, leftover := range []string{missing, missing + ".part"} { + if _, statErr := os.Stat(leftover); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("a failed FetchFile left %s behind: %v", leftover, statErr) + } + } if err = st.Delete(ctx, id); err != nil { t.Fatalf("Delete: %v", err) } - if _, err = st.FetchFile(ctx, id, "disk2.img"); !errors.Is(err, store.ErrNotFound) { + if err = st.FetchFile(ctx, id, "disk2.img", filepath.Join(t.TempDir(), "deleted")); !errors.Is(err, store.ErrNotFound) { t.Fatalf("FetchFile after Delete: %v, want store.ErrNotFound", err) } if _, err = st.ReadMeta(ctx, id); !errors.Is(err, store.ErrNotFound) { From b9bd87e4de41007c4019f550b11cc2c3915bfd5c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:03:43 +0800 Subject: [PATCH 16/28] pool: attach layer blobs through links in the reader's own directory, and log why a read woke the claim cocoon refuses to attach any path under its own root, run and log directories, and layer blobs always live in /oci/blobs, so no reader was ever built and every read silently woke the claim. The blobs are now hard-linked into the reader dir (a CloneFile copy where the link fails) and the links are attached. A link is safe: the blob is immutable, the attach is read-only, and removing the reader dir only drops the link. The relay logs a warning, with the claim id and the error, when it falls back to a wake. The fake engines refuse a path under a configured cocoon root, as cocoon does, so the tests fail if an attach regresses to a blob path. --- docs/deploy.md | 6 ++- e2e/fakeengine_test.go | 16 ++++++- sandboxd/pool/pool_test.go | 24 +++++++++++ sandboxd/pool/reader.go | 10 ++++- sandboxd/pool/reader_test.go | 77 ++++++++++++++++++++++++++++++++-- sandboxd/server/reader.go | 1 + sandboxd/server/reader_test.go | 43 ++++++++++++++++++- 7 files changed, 168 insertions(+), 9 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index c32ffc10..3ef556a5 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -195,8 +195,10 @@ never shared: it serves one claim, and only the claim's own token reaches it. The guest's own kernel does the parsing, as it does for the claim. A reader is built per claim from the claim's hibernate snapshot (or archive record): -- the claim's layer blobs, read from the snapshot's sidecar, are attached to - the reader read-only, and its copy-on-write disk as a private copy that the +- the claim's layer blobs, read from the snapshot's sidecar, are hard-linked + into the reader's directory under `data_dir` (a copy where the link fails, + such as across filesystems; cocoon refuses to attach a path inside its own + directories) and attached read-only, and its copy-on-write disk as a private copy that the reader mounts writable, so the guest replays the ext4 journal onto the copy and never onto the original; - the reader mounts the layers and the copy, assembles a read-only overlay at diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index e4366008..37ec0784 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "slices" "strconv" + "strings" "sync" "time" @@ -92,7 +93,10 @@ func (f *fakeEngine) SnapshotExport(_ context.Context, _, toDir string) error { if err := os.MkdirAll(toDir, 0o750); err != nil { return err } - layer := filepath.Join(f.dir, "layer.erofs") + if err := os.MkdirAll(f.cocoonRoot(), 0o750); err != nil { + return err + } + layer := filepath.Join(f.cocoonRoot(), "layer.erofs") sidecar := `{"storage_configs":[{"path":` + strconv.Quote(layer) + `,"role":"layer"}]}` files := map[string]string{ layer: "erofs", @@ -187,6 +191,12 @@ func (f *fakeEngine) Warmup(context.Context, string, []string) error { return ni func (f *fakeEngine) TrimCow(context.Context, string) error { return nil } func (f *fakeEngine) DiskAttach(_ context.Context, vmName string, spec engine.VolumeSpec) error { + if resolved, err := filepath.EvalSymlinks(spec.Path); err == nil { + root, _ := filepath.EvalSymlinks(f.cocoonRoot()) + if rel, relErr := filepath.Rel(root, resolved); relErr == nil && !strings.HasPrefix(rel, "..") { + return fmt.Errorf("external volume path %s is inside a cocoon-managed directory", spec.Path) + } + } f.mu.Lock() defer f.mu.Unlock() f.volumeVMs[vmName] = true @@ -229,6 +239,10 @@ func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount stri return nil } +func (f *fakeEngine) cocoonRoot() string { + return filepath.Join(f.dir, "cocoon-root") +} + func (f *fakeEngine) volumeOpsLog() []string { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 54f55143..8c4eae31 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1226,6 +1226,7 @@ type fakeEngine struct { startReaderErr error readerDialErr error syncErr error + managedDirs []string caInstalls []string warmups [][]string warmupSocks []string @@ -1617,6 +1618,9 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.Volum if err := ctx.Err(); err != nil { return err } + if err := f.refuseManagedPath(spec.Path); err != nil { + return err + } f.mu.Lock() f.volumeSpecs = append(f.volumeSpecs, spec) f.volumeOps = append(f.volumeOps, "attach:"+spec.Name) @@ -1636,6 +1640,26 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.Volum return err } +func (f *fakeEngine) refuseManagedPath(path string) error { + f.mu.Lock() + dirs := slices.Clone(f.managedDirs) + f.mu.Unlock() + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + resolved = path + } + for _, dir := range dirs { + root, evalErr := filepath.EvalSymlinks(dir) + if evalErr != nil { + root = dir + } + if rel, relErr := filepath.Rel(root, resolved); relErr == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("external volume path %s is inside a cocoon-managed directory", path) + } + } + return nil +} + func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bool) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index 7062fbad..d7c214d4 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -31,6 +31,8 @@ const ( layerRole = "layer" ) +var linkFile = os.Link + type readerSet struct { key types.PoolKey mu sync.Mutex @@ -253,7 +255,13 @@ func (m *Manager) startReader(ctx context.Context, r *reader, layers []string) e disks := engine.ReaderDisks{Cow: readerCowDisk} for i, blob := range layers { name := fmt.Sprintf("rl%d", i) - if err := m.eng.DiskAttach(ctx, r.vm, engine.VolumeSpec{Name: name, Path: blob}); err != nil { + linked := filepath.Join(r.dir, name+".erofs") + if err := linkFile(blob, linked); err != nil { + if err = utils.CloneFile(blob, linked); err != nil { + return fmt.Errorf("link layer %d into the reader dir: %w", i, err) + } + } + if err := m.eng.DiskAttach(ctx, r.vm, engine.VolumeSpec{Name: name, Path: linked}); err != nil { return fmt.Errorf("attach layer %d: %w", i, err) } disks.Layers = append(disks.Layers, name) diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 12f31045..39dc5d4b 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -52,9 +52,17 @@ func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { t.Errorf("reader setup ran %v, want %v", got, want) } cow := eng.volumeSpecs[len(eng.volumeSpecs)-1] - wantSpecs := []engine.VolumeSpec{{Name: "rl0", Path: layers[0]}, {Name: "rl1", Path: layers[1]}, {Name: "rcow", Path: cow.Path, RW: true}} + dir := filepath.Dir(cow.Path) + wantSpecs := []engine.VolumeSpec{{Name: "rl0", Path: filepath.Join(dir, "rl0.erofs")}, {Name: "rl1", Path: filepath.Join(dir, "rl1.erofs")}, {Name: "rcow", Path: cow.Path, RW: true}} if !slices.Equal(eng.volumeSpecs, wantSpecs) { - t.Errorf("attached %+v, want read-only layers then the writable copy %+v", eng.volumeSpecs, wantSpecs) + t.Errorf("attached %+v, want read-only layers linked into the reader dir, then the writable copy %+v", eng.volumeSpecs, wantSpecs) + } + for i, blob := range layers { + linked, statErr := os.Stat(wantSpecs[i].Path) + orig, origErr := os.Stat(blob) + if statErr != nil || origErr != nil || !os.SameFile(linked, orig) { + t.Errorf("layer %d attached as %s, want a hard link of %s: %v %v", i, wantSpecs[i].Path, blob, statErr, origErr) + } } if want := (engine.ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}); len(eng.startedReaders) != 1 || !slices.Equal(eng.startedReaders[0].Layers, want.Layers) || eng.startedReaders[0].Cow != want.Cow { t.Errorf("view started with %+v, want %+v", eng.startedReaders, want) @@ -62,14 +70,74 @@ func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { if got, readErr := os.ReadFile(cow.Path); readErr != nil || string(got) != "disk-bytes" { t.Errorf("reader disk %q, %v, want the snapshot's copy-on-write bytes", got, readErr) } - if entries, _ := os.ReadDir(filepath.Dir(cow.Path)); len(entries) != 2 { - t.Errorf("reader dir holds %d entries, want only the disk and its sidecar", len(entries)) + entries, _ := os.ReadDir(dir) + names := make([]string, len(entries)) + for i, entry := range entries { + names[i] = entry.Name() + } + if want := []string{"cocoon.json", "cow.raw", "rl0.erofs", "rl1.erofs"}; !slices.Equal(names, want) { + t.Errorf("reader dir holds %v, want only %v", names, want) } if n := len(m.pools[readerKey].warm); n != 0 { t.Errorf("reader pool holds %d warm VMs, want the one reader taken", n) } } +func TestReaderLayersFallBackToACopyWhenTheLinkFails(t *testing.T) { + linkFile = func(string, string) error { return errors.New("cross-device link") } + defer func() { linkFile = os.Link }() + eng := newFakeEngine() + layers := readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + + conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + _ = conn.Close() + for i, blob := range layers { + attached := eng.volumeSpecs[i].Path + copied, copyErr := os.Stat(attached) + orig, origErr := os.Stat(blob) + if copyErr != nil || origErr != nil || os.SameFile(copied, orig) { + t.Errorf("layer %d attached as %s, want a copy distinct from %s: %v %v", i, attached, blob, copyErr, origErr) + } + if got, _ := os.ReadFile(attached); string(got) != "erofs" { + t.Errorf("layer %d copy holds %q, want the blob's bytes", i, got) + } + } +} + +func TestReaderReleaseRemovesItsLayerLinksAndKeepsTheBlobs(t *testing.T) { + eng := newFakeEngine() + layers := readerExport(t, eng) + m := newReaderManager(t, eng) + seedReaderVMs(m, "sbx-reader-1") + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + _ = conn.Close() + + m.readers.mu.Lock() + live := m.readers.live[sb.ID] + m.readers.mu.Unlock() + m.dropReader(t.Context(), live, true) + for _, blob := range layers { + if got, readErr := os.ReadFile(blob); readErr != nil || string(got) != "erofs" { + t.Errorf("release damaged the shared blob %s: %q, %v", blob, got, readErr) + } + } + if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { + t.Errorf("release left %d entries under %s", len(entries), readerDir) + } +} + func TestReaderCallsShareOneSetup(t *testing.T) { eng := newFakeEngine() readerExport(t, eng) @@ -691,6 +759,7 @@ func readerExport(t *testing.T, eng *fakeEngine) []string { } sidecar := fmt.Sprintf(`{"storage_configs":[{"path":%q,"role":"layer"},{"path":%q,"role":"layer"},{"path":"/run/cow.raw","role":"cow"},{"path":"/run/cidata.img","role":"cidata"}]}`, layers[0], layers[1]) eng.exportFiles = map[string]string{"cow.raw": "disk-bytes", "cocoon.json": sidecar, "memory-ranges": "guest-memory"} + eng.managedDirs = []string{dir} return layers } diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index ae948bc9..411faf63 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -92,6 +92,7 @@ func (r *router) run() (net.Conn, func(), bool) { r.fail(err) return nil, nil, false default: + log.WithFunc("server.router.run").Warnf(r.ctx, "no reader for %s, waking it: %v", r.id, err) return r.handover(head) } default: diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go index 24c77a7a..52248f9d 100644 --- a/sandboxd/server/reader_test.go +++ b/sandboxd/server/reader_test.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "errors" + "fmt" "io" "net" "net/http/httptest" @@ -14,6 +15,9 @@ import ( "testing" "time" + "github.com/projecteru2/core/log" + "github.com/rs/zerolog" + "github.com/cocoonstack/sandbox/protocol/wire" "github.com/cocoonstack/sandbox/sandboxd/pool" ) @@ -153,8 +157,9 @@ func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing } func TestSleepingUnavailableReaderWakes(t *testing.T) { + logs := captureLogs(t) env := newSleepingEnv(t) - env.reader.setDialErr(pool.ErrNoReader) + env.reader.setDialErr(fmt.Errorf("%w: attach layer 0: boom", pool.ErrNoReader)) conn, r := env.upgrade(t) send(t, conn, statFrame) @@ -162,6 +167,15 @@ func TestSleepingUnavailableReaderWakes(t *testing.T) { if wakes, _ := env.reader.counts(); wakes != 1 { t.Errorf("wakes=%d, want the read to fall back to a wake", wakes) } + got := logs.String() + for _, want := range []string{`"level":"warn"`, "sb_1", "attach layer 0: boom"} { + if !strings.Contains(got, want) { + t.Errorf("log %q lacks %q: the fallback to a wake must be visible", got, want) + } + } + if n := strings.Count(got, "waking it"); n != 1 { + t.Errorf("%d warnings for one failed setup, want 1", n) + } } func TestSleepingFailuresAnswerWithAnErrorFrame(t *testing.T) { @@ -273,6 +287,33 @@ func TestSleepingCloseRelaysEndsReaderConnections(t *testing.T) { } } +func captureLogs(t *testing.T) *logSink { + t.Helper() + sink := &logSink{} + logger := log.GetGlobalLogger() + prev := *logger + *logger = zerolog.New(sink) + t.Cleanup(func() { *logger = prev }) + return sink +} + +type logSink struct { + mu sync.Mutex + buf strings.Builder +} + +func (l *logSink) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.Write(p) +} + +func (l *logSink) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.String() +} + type sleepingEnv struct { ts *httptest.Server srv *Server From 9da2bbdbb49be112ceb88feaa199ac83774b8eb2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:04:24 +0800 Subject: [PATCH 17/28] review: place the reader type blocks after the Manager method set Layout only: the exported Manager methods lead the file and the readerSet and reader types follow the last method, as the type-block layout rules require; the fake engine's unexported helper moves below its exported methods. --- sandboxd/pool/pool_test.go | 40 +++++++++++++++++++------------------- sandboxd/pool/reader.go | 34 ++++++++++++++++---------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 8c4eae31..083a5ea3 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1640,26 +1640,6 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.Volum return err } -func (f *fakeEngine) refuseManagedPath(path string) error { - f.mu.Lock() - dirs := slices.Clone(f.managedDirs) - f.mu.Unlock() - resolved, err := filepath.EvalSymlinks(path) - if err != nil { - resolved = path - } - for _, dir := range dirs { - root, evalErr := filepath.EvalSymlinks(dir) - if evalErr != nil { - root = dir - } - if rel, relErr := filepath.Rel(root, resolved); relErr == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return fmt.Errorf("external volume path %s is inside a cocoon-managed directory", path) - } - } - return nil -} - func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bool) error { f.mu.Lock() defer f.mu.Unlock() @@ -1732,6 +1712,26 @@ func (f *fakeEngine) DialReader(_ context.Context, vsockSocket string) (net.Conn return conn, nil } +func (f *fakeEngine) refuseManagedPath(path string) error { + f.mu.Lock() + dirs := slices.Clone(f.managedDirs) + f.mu.Unlock() + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + resolved = path + } + for _, dir := range dirs { + root, evalErr := filepath.EvalSymlinks(dir) + if evalErr != nil { + root = dir + } + if rel, relErr := filepath.Rel(root, resolved); relErr == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("external volume path %s is inside a cocoon-managed directory", path) + } + } + return nil +} + func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index d7c214d4..fe839c2a 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -33,23 +33,6 @@ const ( var linkFile = os.Link -type readerSet struct { - key types.PoolKey - mu sync.Mutex - live map[string]*reader - flights singleflight.Group -} - -type reader struct { - claim string - source string - vm string - sock string - dir string - conns int - idle *time.Timer -} - // DialReader opens a silkd connection to a sleeping claim's reader VM without waking the claim. func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Conn, source string, err error) { sb, ok := m.claim(id, token) @@ -279,6 +262,23 @@ func (m *Manager) destroyReader(ctx context.Context, r *reader) { } } +type readerSet struct { + key types.PoolKey + mu sync.Mutex + live map[string]*reader + flights singleflight.Group +} + +type reader struct { + claim string + source string + vm string + sock string + dir string + conns int + idle *time.Timer +} + // sleepSource names a settled sleep's snapshot or record; callers hold m.mu. func sleepSource(sb *types.Sandbox) string { if sb.Failed != "" || sb.PendingSnap != "" || !sb.ReaderSafe { From 544dc8943ac98d40933cc21053a081933e34b893 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:05:27 +0800 Subject: [PATCH 18/28] review: split the FetchFile checks out of RunContract Keeps the contract under the cyclomatic limit; behavior unchanged. --- sandboxd/store/storetest/storetest.go | 38 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index 60a05f01..b0d9ac3d 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -77,22 +77,7 @@ func RunContract(t *testing.T, st store.Store) { t.Errorf("second-generation export: %q, %v", got, readErr) } - dst := filepath.Join(t.TempDir(), "fetched") - if err = st.FetchFile(ctx, id, "disk2.img", dst); err != nil { - t.Fatalf("FetchFile: %v", err) - } - if got, readErr := os.ReadFile(dst); readErr != nil || string(got) != "second-gen" { //nolint:gosec // test path - t.Errorf("fetched file: %q, %v", got, readErr) - } - missing := filepath.Join(t.TempDir(), "missing") - if err = st.FetchFile(ctx, id, "absent.img", missing); !errors.Is(err, store.ErrNotFound) { - t.Errorf("FetchFile of an absent file: %v, want store.ErrNotFound", err) - } - for _, leftover := range []string{missing, missing + ".part"} { - if _, statErr := os.Stat(leftover); !errors.Is(statErr, os.ErrNotExist) { - t.Errorf("a failed FetchFile left %s behind: %v", leftover, statErr) - } - } + runFetchFileContract(t, st, id) if err = st.Delete(ctx, id); err != nil { t.Fatalf("Delete: %v", err) @@ -113,6 +98,27 @@ func RunContract(t *testing.T, st store.Store) { runLabelsContract(t, st) } +func runFetchFileContract(t *testing.T, st store.Store, id string) { + t.Helper() + ctx := t.Context() + dst := filepath.Join(t.TempDir(), "fetched") + if err := st.FetchFile(ctx, id, "disk2.img", dst); err != nil { + t.Fatalf("FetchFile: %v", err) + } + if got, readErr := os.ReadFile(dst); readErr != nil || string(got) != "second-gen" { //nolint:gosec // test path + t.Errorf("fetched file: %q, %v", got, readErr) + } + missing := filepath.Join(t.TempDir(), "missing") + if err := st.FetchFile(ctx, id, "absent.img", missing); !errors.Is(err, store.ErrNotFound) { + t.Errorf("FetchFile of an absent file: %v, want store.ErrNotFound", err) + } + for _, leftover := range []string{missing, missing + ".part"} { + if _, statErr := os.Stat(leftover); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("a failed FetchFile left %s behind: %v", leftover, statErr) + } + } +} + func runDigestContract(t *testing.T, st store.Store) { t.Helper() ctx := t.Context() From aff2e8aad4e0959df10856a8b898e89169489151 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 12:54:15 +0800 Subject: [PATCH 19/28] reader: attach the sleeping claim's disk through cocoon's vm disk attach The reader VM is now a warm VM of the claim's own pool, so it boots the claim's image and layers. A hibernated claim's disk comes from --from-snapshot, an archived claim's from --from-dir over snapshot.json and the files its disk_attach_files lists. The guest mounts the attached disk by the returned serial and the layers from its own cocoon.layers= cmdline. sandboxd no longer reads cocoon file names, the sidecar schema or layer blob paths. reader becomes a per-pool, reloadable, config-owned flag; the single reader-pool setting is removed. --- e2e/fakeengine_test.go | 50 ++--- e2e/reader_test.go | 21 +-- e2e/wire_parity_test.go | 2 +- sandboxd/config/config.go | 32 +--- sandboxd/config/config_test.go | 18 +- sandboxd/config/reload.go | 1 + sandboxd/config/reload_test.go | 6 +- sandboxd/engine/reader.go | 43 +++-- sandboxd/engine/reader_test.go | 61 +++++-- sandboxd/engine/volume.go | 25 +++ sandboxd/engine/volume_test.go | 39 ++++ sandboxd/pool/hibernate.go | 6 +- sandboxd/pool/pool.go | 8 +- sandboxd/pool/pool_test.go | 63 ++++--- sandboxd/pool/poolstore.go | 3 +- sandboxd/pool/poolstore_test.go | 1 + sandboxd/pool/reader.go | 161 ++++++++--------- sandboxd/pool/reader_test.go | 311 ++++++++++++++++---------------- sandboxd/pool/setpools.go | 3 + sandboxd/pool/view.go | 5 + sandboxd/pool/view_test.go | 1 + 21 files changed, 464 insertions(+), 396 deletions(-) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 37ec0784..9bbc9113 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -9,8 +9,6 @@ import ( "os" "path/filepath" "slices" - "strconv" - "strings" "sync" "time" @@ -31,8 +29,9 @@ type fakeEngine struct { guestEnvs map[string]string seq int - readerRoot string - readerStarts []string + readerRoot string + readerStarts []string + readerAttaches []string } func newFakeEngine(dir string) *fakeEngine { @@ -90,28 +89,21 @@ func (f *fakeEngine) ReconcileStaleCreate(context.Context, string) (engine.Stale func (f *fakeEngine) SnapshotSave(_ context.Context, _, _ string) error { return nil } func (f *fakeEngine) SnapshotExport(_ context.Context, _, toDir string) error { - if err := os.MkdirAll(toDir, 0o750); err != nil { - return err - } - if err := os.MkdirAll(f.cocoonRoot(), 0o750); err != nil { - return err - } - layer := filepath.Join(f.cocoonRoot(), "layer.erofs") - sidecar := `{"storage_configs":[{"path":` + strconv.Quote(layer) + `,"role":"layer"}]}` - files := map[string]string{ - layer: "erofs", - filepath.Join(toDir, "cow.raw"): "disk", - filepath.Join(toDir, "cocoon.json"): sidecar, - } - for path, body := range files { - if err := os.WriteFile(path, []byte(body), 0o600); err != nil { - return err - } - } - return nil + return os.MkdirAll(toDir, 0o750) +} + +func (f *fakeEngine) DiskAttachSnapshot(_ context.Context, vmName, snapName string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.readerAttaches = append(f.readerAttaches, vmName+":"+snapName) + return "snap-" + snapName, nil +} + +func (f *fakeEngine) DiskAttachDir(context.Context, string, string) (string, error) { + return "", errors.New("no archive in this fake") } -func (f *fakeEngine) StartReader(_ context.Context, vsockSocket string, _ engine.ReaderDisks) error { +func (f *fakeEngine) StartReader(_ context.Context, vsockSocket, _ string) error { f.mu.Lock() defer f.mu.Unlock() f.readerStarts = append(f.readerStarts, vsockSocket) @@ -191,12 +183,6 @@ func (f *fakeEngine) Warmup(context.Context, string, []string) error { return ni func (f *fakeEngine) TrimCow(context.Context, string) error { return nil } func (f *fakeEngine) DiskAttach(_ context.Context, vmName string, spec engine.VolumeSpec) error { - if resolved, err := filepath.EvalSymlinks(spec.Path); err == nil { - root, _ := filepath.EvalSymlinks(f.cocoonRoot()) - if rel, relErr := filepath.Rel(root, resolved); relErr == nil && !strings.HasPrefix(rel, "..") { - return fmt.Errorf("external volume path %s is inside a cocoon-managed directory", spec.Path) - } - } f.mu.Lock() defer f.mu.Unlock() f.volumeVMs[vmName] = true @@ -239,10 +225,6 @@ func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount stri return nil } -func (f *fakeEngine) cocoonRoot() string { - return filepath.Join(f.dir, "cocoon-root") -} - func (f *fakeEngine) volumeOpsLog() []string { f.mu.Lock() defer f.mu.Unlock() diff --git a/e2e/reader_test.go b/e2e/reader_test.go index 185cce61..1266683d 100644 --- a/e2e/reader_test.go +++ b/e2e/reader_test.go @@ -8,15 +8,11 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/pool" - "github.com/cocoonstack/sandbox/sandboxd/types" ) -var readerKey = types.PoolKey{Template: "reader:24.04", Net: types.NetNone, Size: types.SizeSmall} - func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { st := startConfigStack(t, "node-token", &config.Config{ - Reader: &config.ReaderSpec{Template: readerKey.Template}, - Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}, {PoolKey: readerKey, Warm: 1}}, + Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 2, Reader: true}}, }) root := t.TempDir() if err := os.MkdirAll(filepath.Join(root, "work"), 0o750); err != nil { @@ -28,11 +24,6 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { st.eng.mu.Lock() st.eng.readerRoot = root st.eng.mu.Unlock() - waitFor(t, func() bool { - pools, _ := st.mgr.Info() - return slices.ContainsFunc(pools, func(p pool.PoolInfo) bool { return p.Key == readerKey && p.Warm >= 1 }) - }) - sb, err := st.client.New(t.Context(), testKey.Template) if err != nil { t.Fatalf("New: %v", err) @@ -41,6 +32,10 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { if err = sb.Hibernate(t.Context()); err != nil { t.Fatalf("Hibernate: %v", err) } + waitFor(t, func() bool { + pools, _ := st.mgr.Info() + return slices.ContainsFunc(pools, func(p pool.PoolInfo) bool { return p.Key == testKey && p.Warm >= 1 }) + }) got, err := sb.ReadFile(t.Context(), "/work/out.txt") if err != nil || string(got) != "result" { @@ -50,10 +45,10 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { t.Fatal("reading a file woke the claim") } st.eng.mu.Lock() - started := len(st.eng.readerStarts) + started, attached := len(st.eng.readerStarts), len(st.eng.readerAttaches) st.eng.mu.Unlock() - if started != 1 { - t.Errorf("%d readers started, want one", started) + if started != 1 || attached != 1 { + t.Errorf("%d readers started over %d snapshot copies, want one of each", started, attached) } if out, execErr := sb.Exec(t.Context(), "echo", "woke"); execErr != nil || out != "woke\n" { diff --git a/e2e/wire_parity_test.go b/e2e/wire_parity_test.go index 6e507ad1..012f172e 100644 --- a/e2e/wire_parity_test.go +++ b/e2e/wire_parity_test.go @@ -59,7 +59,7 @@ var wirePairs = []struct { {"SandboxSummary", false, pool.SandboxSummary{}, []string{"token"}}, {"sandboxListResponse", false, server.SandboxListResponse{}, nil}, {"poolUpdate", true, server.PoolUpdateRequest{}, nil}, - {"PoolSpec", true, config.PoolSpec{}, []string{"egress", "warmup", "egress_upstream_env", "capture_trim", "storage", "replicate_seconds", "net", "size"}}, + {"PoolSpec", true, config.PoolSpec{}, []string{"egress", "warmup", "egress_upstream_env", "capture_trim", "storage", "replicate_seconds", "reader", "net", "size"}}, {"tenantsUpdate", true, server.TenantsRequest{}, nil}, {"tenantUpdate", true, server.TenantRequest{}, nil}, {"TenantSpec", true, reflect.Zero(reflect.TypeOf(server.TenantsRequest{}.Tenants).Elem()).Interface(), []string{"token"}}, diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index c678a588..fba3efba 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -53,6 +53,8 @@ type PoolSpec struct { Storage string `json:"storage,omitempty"` // ReplicateSeconds snapshots each running claim of the pool to the checkpoint store at this interval; zero turns replication off. ReplicateSeconds int `json:"replicate_seconds,omitzero"` + // Reader serves read-only file calls on this pool's hibernated or archived claims from a warm VM, without waking them. + Reader bool `json:"reader,omitzero"` IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitzero"` ArchiveAfterSeconds int `json:"archive_after_seconds,omitzero"` @@ -93,6 +95,9 @@ func (s PoolSpec) ValidateLimits() error { if s.Net == types.NetEgress && s.IdleHibernateSeconds > 0 { return fmt.Errorf("idle_hibernate_seconds is not supported for egress pools") } + if s.Reader && s.Warm < 1 { + return fmt.Errorf("reader needs warm of at least 1") + } if slices.Contains(s.Warmup, "") { return fmt.Errorf("warmup must not contain an empty argument") } @@ -164,16 +169,6 @@ type VolumeSpec struct { Tenants []string `json:"tenants,omitempty"` } -// ReaderSpec names the none-lane pool whose warm VMs serve read-only file calls on a hibernated or archived claim. -type ReaderSpec struct { - Template string `json:"template"` - Size types.Size `json:"size,omitempty"` -} - -func (r ReaderSpec) Key() types.PoolKey { - return types.PoolKey{Template: r.Template, Size: r.Size}.Defaulted() -} - // DiskReserve refuses claims, wakes and refills while a listed directory's filesystem has less than Free left. type DiskReserve struct { Free string `json:"free"` @@ -278,7 +273,6 @@ type Config struct { AuditLog bool `json:"audit_log,omitzero"` DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` - Reader *ReaderSpec `json:"reader,omitempty"` Volumes []VolumeSpec `json:"volumes,omitempty"` Mesh *MeshConfig `json:"mesh,omitempty"` @@ -377,7 +371,7 @@ func (c *Config) validate() error { if c.CheckpointPeerHeal && c.APIToken == "" { return fmt.Errorf("checkpoint_peer_heal requires api_token: without it resolveScope leaves the raw checkpoint blob GET reachable with no credential") } - for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateReader, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { + for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { if err := check(); err != nil { return err } @@ -624,20 +618,6 @@ func (c *Config) validateDiskReserve() error { return nil } -func (c *Config) validateReader() error { - if c.Reader == nil { - return nil - } - key := c.Reader.Key() - if err := key.Validate(); err != nil { - return fmt.Errorf("reader: %w", err) - } - if !slices.ContainsFunc(c.Pools, func(p PoolSpec) bool { return p.PoolKey == key && p.Warm > 0 }) { - return fmt.Errorf("reader %s %s/%s must be a configured pool with warm of at least 1", key.Template, key.Net, key.Size) - } - return nil -} - func (c *Config) validateMetaStore() error { ms := c.MetaStore switch { diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 36540738..06c4fe3c 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -167,11 +167,7 @@ func TestLoadRejectsInvalid(t *testing.T) { {"duplicate volume name", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img"},{"name":"data","path":"/srv/datasets/b.img"}]}`, "duplicate volume"}, {"relative volume path", `{"pools":[],"volumes":[{"name":"data","path":"datasets/a.img"}]}`, "path must be absolute"}, {"bad volume directio", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img","directio":"yes"}]}`, "directio must be"}, - {"reader without a pool", `{"pools":[{"template":"rt:24.04"}],"reader":{"template":"reader:24.04"}}`, "must be a configured pool"}, - {"reader on another size", `{"pools":[{"template":"rt:24.04"}],"reader":{"template":"rt:24.04","size":"large"}}`, "must be a configured pool"}, - {"reader pool without warm", `{"pools":[{"template":"rt:24.04","warm":0}],"reader":{"template":"rt:24.04"}}`, "warm of at least 1"}, - {"reader without a template", `{"pools":[],"reader":{}}`, "reader: template"}, - {"reader typo", `{"pools":[{"template":"rt:24.04"}],"reader":{"template":"rt:24.04","sizes":"small"}}`, "unknown object member"}, + {"reader without warm", `{"pools":[{"template":"rt:24.04","warm":0,"reader":true}]}`, "reader needs warm"}, } { t.Run(tt.name, func(t *testing.T) { _, err := Load(writeConfig(t, tt.body)) @@ -285,17 +281,13 @@ func TestLoadAcceptsPreviewAdvertiseShapes(t *testing.T) { } } -func TestLoadAcceptsReaderOnAConfiguredPool(t *testing.T) { - cfg, err := Load(writeConfig(t, `{"pools":[{"template":"reader:24.04","warm":2}],"reader":{"template":"reader:24.04"}}`)) +func TestLoadReadsReader(t *testing.T) { + cfg, err := Load(writeConfig(t, `{"pools":[{"template":"rt:24.04","reader":true},{"template":"py:3.12"}]}`)) if err != nil { t.Fatalf("Load: %v", err) } - want := types.PoolKey{Template: "reader:24.04", Net: types.NetNone, Size: types.SizeSmall} - if got := cfg.Reader.Key(); got != want { - t.Errorf("reader key = %+v, want the defaulted pool key %+v", got, want) - } - if cfg, err = Load(writeConfig(t, `{"pools":[]}`)); err != nil || cfg.Reader != nil { - t.Errorf("a config without reader loaded as %+v, %v, want the feature off", cfg.Reader, err) + if !cfg.Pools[0].Reader || cfg.Pools[1].Reader { + t.Errorf("reader %v/%v, want on for the pool that names it and off by default", cfg.Pools[0].Reader, cfg.Pools[1].Reader) } } diff --git a/sandboxd/config/reload.go b/sandboxd/config/reload.go index 7beb4132..0b6647e4 100644 --- a/sandboxd/config/reload.go +++ b/sandboxd/config/reload.go @@ -62,6 +62,7 @@ func diffPools(cur, next []PoolSpec) (changed, ignored, restart []string) { {"warmup", slices.Equal(x.Warmup, y.Warmup)}, {"capture_trim", x.CaptureTrim == y.CaptureTrim}, {"storage", x.Storage == y.Storage}, + {"reader", x.Reader == y.Reader}, {"egress_upstream_env", x.EgressUpstreamEnv == y.EgressUpstreamEnv}, } { if !f.same { diff --git a/sandboxd/config/reload_test.go b/sandboxd/config/reload_test.go index 905b7370..18e0263d 100644 --- a/sandboxd/config/reload_test.go +++ b/sandboxd/config/reload_test.go @@ -19,7 +19,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { next.EgressInternalAllow = []string{"10.8.0.0/16"} next.Pools = []PoolSpec{ {PoolKey: key, Warm: 5, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "pypi.org"}}}}, - {PoolKey: v13, Warmup: []string{"node", "-e", "0"}, Storage: "40G"}, + {PoolKey: v13, Warmup: []string{"node", "-e", "0"}, Storage: "40G", Reader: true}, } next.EgressClasses = []EgressClass{{Name: "desk", Egress: &egress.Policy{}}} next.EgressConnRate = &EgressConnRate{PerSecond: 30, Burst: 600} @@ -36,6 +36,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { "pools[desktop:v12 none small].egress", "pools[desktop:v13 none small].warmup", "pools[desktop:v13 none small].storage", + "pools[desktop:v13 none small].reader", } if !slices.Equal(changed, wantChanged) { t.Errorf("changed %v, want %v", changed, wantChanged) @@ -56,8 +57,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { restart := *cur restart.Listen, restart.Bridges, restart.EgressUsageBytes = "127.0.0.1:8888", []string{"br1"}, true - restart.Reader = &ReaderSpec{Template: "desktop:v12"} - if _, _, err := cur.ReloadDiff(&restart); err == nil || err.Error() != "listen, bridges, reader change only at a restart" { + if _, _, err := cur.ReloadDiff(&restart); err == nil || err.Error() != "listen, bridges change only at a restart" { t.Errorf("ReloadDiff over restart fields: %v", err) } if changed, ignored, err := cur.ReloadDiff(cur); err != nil || changed != nil || ignored != nil { diff --git a/sandboxd/engine/reader.go b/sandboxd/engine/reader.go index e30b520c..10f5df79 100644 --- a/sandboxd/engine/reader.go +++ b/sandboxd/engine/reader.go @@ -18,41 +18,44 @@ const ( readerSilkd = "/usr/local/bin/silkd" readerStartTimeout = 30 * time.Second readerProbeTimeout = 10 * time.Second + layersParam = "cocoon.layers=" ) -// ReaderDisks names the disks attached to a reader VM: the claim's layers, bottom layer first, and its writable copy-on-write disk. -type ReaderDisks struct { - Layers []string - Cow string -} - -// StartReader mounts the attached disks as a read-only overlay of the claim's filesystem and starts a silkd chrooted into it. -func (e *Engine) StartReader(ctx context.Context, vsockSocket string, d ReaderDisks) error { +// StartReader mounts the attached copy-on-write disk over the layers the VM booted with as a read-only overlay of the claim's filesystem, and starts a silkd chrooted into it. +func (e *Engine) StartReader(ctx context.Context, vsockSocket, cowDisk string) error { ctx, cancel := context.WithTimeout(ctx, readerStartTimeout) defer cancel() + cmdline, err := e.silkdReadFile(ctx, vsockSocket, "/proc/cmdline", silkdReadMax) + if err != nil { + return fmt.Errorf("read guest cmdline: %w", err) + } + layers := cmdlineLayers(string(cmdline)) + if len(layers) == 0 { + return fmt.Errorf("guest cmdline names no %s", layersParam) + } cow := readerRoot + "/cow" - if err := e.MountVolume(ctx, vsockSocket, d.Cow, cow, true); err != nil { + if err = e.MountVolume(ctx, vsockSocket, cowDisk, cow, true); err != nil { return fmt.Errorf("mount claim disk: %w", err) } lower := []string{cow + "/upper"} - for i, layer := range slices.Backward(d.Layers) { + for i, layer := range layers { mount := fmt.Sprintf("%s/layer%d", readerRoot, i) - if err := e.MountVolume(ctx, vsockSocket, layer, mount, false); err != nil { + if err = e.MountVolume(ctx, vsockSocket, layer, mount, false); err != nil { return fmt.Errorf("mount layer %d: %w", i, err) } lower = append(lower, mount) } - if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", readerView); err != nil { + if err = e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", readerView); err != nil { return fmt.Errorf("create view mount point: %w", err) } opts := "ro,metacopy=on,redirect_dir=follow,lowerdir=" + strings.Join(lower, ":") - if err := e.silkdExec(ctx, vsockSocket, "mount", "-t", "overlay", "-o", opts, "--", "overlay", readerView); err != nil { + if err = e.silkdExec(ctx, vsockSocket, "mount", "-t", "overlay", "-o", opts, "--", "overlay", readerView); err != nil { return fmt.Errorf("mount view: %w", err) } - if err := e.silkdExec(ctx, vsockSocket, "test", "-x", readerView+readerSilkd); err != nil { + if err = e.silkdExec(ctx, vsockSocket, "test", "-x", readerView+readerSilkd); err != nil { return fmt.Errorf("claim image has no silkd at %s: %w", readerSilkd, err) } - if err := e.silkdSpawn(ctx, vsockSocket, "env", fmt.Sprintf("SILKD_PORT=%d", readerPort), "chroot", readerView, readerSilkd); err != nil { + if err = e.silkdSpawn(ctx, vsockSocket, "env", fmt.Sprintf("SILKD_PORT=%d", readerPort), "chroot", readerView, readerSilkd); err != nil { return fmt.Errorf("start view silkd: %w", err) } return e.probe(ctx, vsockSocket, readerPort, readerProbeTimeout) @@ -85,3 +88,13 @@ func (e *Engine) silkdSpawn(ctx context.Context, vsockSocket string, argv ...str return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) } } + +func cmdlineLayers(cmdline string) []string { + var layers []string + for field := range strings.FieldsSeq(cmdline) { + if val, ok := strings.CutPrefix(field, layersParam); ok { + layers = slices.DeleteFunc(strings.Split(val, ","), func(s string) bool { return s == "" }) + } + } + return layers +} diff --git a/sandboxd/engine/reader_test.go b/sandboxd/engine/reader_test.go index 3fd51ffa..c72d8261 100644 --- a/sandboxd/engine/reader_test.go +++ b/sandboxd/engine/reader_test.go @@ -8,24 +8,24 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -func TestStartReaderAssemblesTheViewTopLayerFirst(t *testing.T) { +func TestStartReaderOverlaysTheAttachedDiskOnTheGuestsOwnLayers(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) configureReaderDevices(fake) e := New("cocoon", nil, nil, false, false, "") - if err := e.StartReader(t.Context(), sock, ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}); err != nil { + if err := e.StartReader(t.Context(), sock, "snap-ab12cd34"); err != nil { t.Fatalf("StartReader: %v", err) } want := [][]string{ {"mkdir", "-p", "--", "/run/reader/cow"}, {"mount", "-o", "rw", "--", "/dev/vdb", "/run/reader/cow"}, - {"mkdir", "-p", "--", "/run/reader/layer1"}, - {"mount", "-o", "ro", "--", "/dev/vdd", "/run/reader/layer1"}, {"mkdir", "-p", "--", "/run/reader/layer0"}, {"mount", "-o", "ro", "--", "/dev/vdc", "/run/reader/layer0"}, + {"mkdir", "-p", "--", "/run/reader/layer1"}, + {"mount", "-o", "ro", "--", "/dev/vdd", "/run/reader/layer1"}, {"mkdir", "-p", "--", "/view"}, - {"mount", "-t", "overlay", "-o", "ro,metacopy=on,redirect_dir=follow,lowerdir=/run/reader/cow/upper:/run/reader/layer1:/run/reader/layer0", "--", "overlay", "/view"}, + {"mount", "-t", "overlay", "-o", "ro,metacopy=on,redirect_dir=follow,lowerdir=/run/reader/cow/upper:/run/reader/layer0:/run/reader/layer1", "--", "overlay", "/view"}, {"test", "-x", "/view/usr/local/bin/silkd"}, {"env", "SILKD_PORT=2051", "chroot", "/view", "/usr/local/bin/silkd"}, } @@ -57,7 +57,7 @@ func TestStartReaderFailsFastWhenTheClaimImageHasNoSilkd(t *testing.T) { fake.execFailAt, fake.execCode = 9, 1 fake.mu.Unlock() - err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}) + err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, "snap-ab12cd34") if err == nil || !strings.Contains(err.Error(), "no silkd") { t.Fatalf("StartReader = %v, want the missing silkd reported", err) } @@ -73,17 +73,55 @@ func TestStartReaderStopsAtTheFirstFailedMount(t *testing.T) { fake := serveFakeSilkd(t, sock) configureReaderDevices(fake) fake.mu.Lock() - fake.execFailAt, fake.execCode = 4, 32 + fake.execFailAt, fake.execCode = 6, 32 fake.mu.Unlock() - err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}) + err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, "snap-ab12cd34") if err == nil || !strings.Contains(err.Error(), "mount layer 1") { t.Fatalf("StartReader = %v, want the layer 1 mount failure", err) } fake.mu.Lock() defer fake.mu.Unlock() - if len(fake.execCalls) != 4 { - t.Errorf("guest ran %d commands after the failed mount, want it to stop at 4: %v", len(fake.execCalls), fake.execCalls) + if len(fake.execCalls) != 6 { + t.Errorf("guest ran %d commands after the failed mount, want it to stop at 6: %v", len(fake.execCalls), fake.execCalls) + } +} + +func TestStartReaderRefusesAGuestWithoutLayers(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureReaderDevices(fake) + fake.mu.Lock() + fake.serial["/proc/cmdline"] = []byte("console=hvc0 cocoon.cow=cow\n") + fake.mu.Unlock() + + err := New("cocoon", nil, nil, false, false, "").StartReader(t.Context(), sock, "snap-ab12cd34") + if err == nil || !strings.Contains(err.Error(), "cocoon.layers=") { + t.Fatalf("StartReader = %v, want the missing layers reported", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != 0 { + t.Errorf("guest ran %v, want nothing mounted", fake.execCalls) + } +} + +func TestCmdlineLayersReadsTheLastParamInOrder(t *testing.T) { + for _, tt := range []struct { + name string + cmdline string + want []string + }{ + {"in order", "console=hvc0 cocoon.layers=l0,l1,l2 cocoon.cow=cow\n", []string{"l0", "l1", "l2"}}, + {"last param wins", "cocoon.layers=old cocoon.layers=l0,l1", []string{"l0", "l1"}}, + {"empty entries dropped", "cocoon.layers=a,,b,", []string{"a", "b"}}, + {"absent", "console=hvc0 cocoon.cow=cow", nil}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := cmdlineLayers(tt.cmdline); !slices.Equal(got, tt.want) { + t.Errorf("cmdlineLayers = %v, want %v", got, tt.want) + } + }) } } @@ -111,8 +149,9 @@ func configureReaderDevices(f *fakeSilkd) { {Name: "vdc", Kind: wire.FileKindSymlink}, {Name: "vdd", Kind: wire.FileKindSymlink}, } - for dev, serial := range map[string]string{"vda": "root", "vdb": "rcow", "vdc": "rl0", "vdd": "rl1"} { + for dev, serial := range map[string]string{"vda": "root", "vdb": "snap-ab12cd34", "vdc": "l0", "vdd": "l1"} { f.serial["/sys/block/"+dev+"/serial"] = []byte(serial + "\n") f.stat["/dev/"+dev] = true } + f.serial["/proc/cmdline"] = []byte("console=hvc0 boot=cocoon-overlay cocoon.layers=l0,l1 cocoon.cow=cow\n") } diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index 2cc405c2..637cc7dc 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -3,6 +3,7 @@ package engine import ( "cmp" "context" + "encoding/json/v2" "errors" "fmt" "strings" @@ -49,6 +50,16 @@ func (e *Engine) DiskAttach(ctx context.Context, vmName string, spec VolumeSpec) return err } +// DiskAttachSnapshot hot-attaches a private read-write copy of a local snapshot's copy-on-write disk and returns its guest serial. +func (e *Engine) DiskAttachSnapshot(ctx context.Context, vmName, snapName string) (string, error) { + return e.diskAttachCopy(ctx, vmName, "--from-snapshot", snapName) +} + +// DiskAttachDir is DiskAttachSnapshot from an export directory. +func (e *Engine) DiskAttachDir(ctx context.Context, vmName, dir string) (string, error) { + return e.diskAttachCopy(ctx, vmName, "--from-dir", dir) +} + // MountVolume discovers and mounts a hot-attached disk at mount, read-only unless rw. func (e *Engine) MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error { ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout) @@ -140,6 +151,20 @@ func (e *Engine) mountDevice(ctx context.Context, vsockSocket, device, name, mou return nil } +func (e *Engine) diskAttachCopy(ctx context.Context, vmName, sourceFlag, source string) (string, error) { + out, err := e.run(ctx, "vm", "disk", "attach", vmName, sourceFlag, source, argOutput, formatJSON) + if err != nil { + return "", err + } + var attached struct { + Name string `json:"name"` + } + if err = json.Unmarshal(out, &attached); err != nil || attached.Name == "" { + return "", fmt.Errorf("parse disk attach result %q: %w", out, cmp.Or(err, errors.New("no disk name"))) + } + return attached.Name, nil +} + func (e *Engine) diskAttachArgs(vmName string, spec VolumeSpec) ([]string, error) { directIO, err := spec.directIO() if err != nil { diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index e41c5c82..62ad7ce7 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -3,6 +3,8 @@ package engine import ( "context" "errors" + "os" + "path/filepath" "slices" "strings" "testing" @@ -54,6 +56,43 @@ func TestDiskAttachArgsRejectBadOptions(t *testing.T) { } } +func TestDiskAttachCopyPassesTheSourceAndReturnsTheGuestSerial(t *testing.T) { + for _, tt := range []struct { + name string + attach func(*Engine) (string, error) + wantArgs string + }{ + {"snapshot", func(e *Engine) (string, error) { + return e.DiskAttachSnapshot(t.Context(), "sbx-1", "snap-1") + }, "vm disk attach sbx-1 --from-snapshot snap-1 --output json"}, + {"directory", func(e *Engine) (string, error) { + return e.DiskAttachDir(t.Context(), "sbx-1", "/srv/export") + }, "vm disk attach sbx-1 --from-dir /srv/export --output json"}, + } { + t.Run(tt.name, func(t *testing.T) { + args := filepath.Join(t.TempDir(), "args") + t.Setenv("COCOON_TEST_ARGS", args) + fakeCocoon(t, "#!/bin/sh\necho \"$*\" > \"$COCOON_TEST_ARGS\"\necho '{\"vm\":\"sbx-1\",\"name\":\"snap-ab12cd34\",\"id\":\"d1\",\"format\":\"raw\"}'\n") + name, err := tt.attach(New("cocoon", nil, nil, false, false, "")) + if err != nil || name != "snap-ab12cd34" { + t.Fatalf("attach = %q, %v, want the serial cocoon reported", name, err) + } + if got, _ := os.ReadFile(args); strings.TrimSpace(string(got)) != tt.wantArgs { + t.Errorf("cocoon ran %q, want %q", got, tt.wantArgs) + } + }) + } +} + +func TestDiskAttachCopyRejectsAnUnnamedResult(t *testing.T) { + for _, body := range []string{"not json", `{"vm":"sbx-1"}`} { + fakeCocoon(t, "#!/bin/sh\necho '"+body+"'\n") + if name, err := New("cocoon", nil, nil, false, false, "").DiskAttachSnapshot(t.Context(), "sbx-1", "snap-1"); err == nil { + t.Errorf("DiskAttachSnapshot over %q = %q, want an error", body, name) + } + } +} + func TestMountVolumeUsesSysfsAndRequestedMode(t *testing.T) { for _, tt := range []struct { name string diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index d192b43c..cac827ed 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -29,9 +29,9 @@ func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string return m.agentSocket(ctx, id, token, false) } -// WakeAgentSocketOrRead is WakeAgentSocket, except that with a reader a sleeping claim answers ErrSleeping instead of waking. +// WakeAgentSocketOrRead is WakeAgentSocket, except that a sleep a reader can serve answers ErrSleeping instead of waking. func (m *Manager) WakeAgentSocketOrRead(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) { - return m.agentSocket(ctx, id, token, m.readers != nil) + return m.agentSocket(ctx, id, token, true) } func (m *Manager) agentSocket(ctx context.Context, id, token string, readerOK bool) (string, <-chan struct{}, func(), error) { @@ -101,7 +101,7 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return fmt.Errorf("hibernate %s: persist intent: %w", sb.ID, err) } readerSafe := false - if m.readers != nil { + if m.view.Load().poolReaders[sb.Key] { err := m.eng.SyncGuest(ctx, sb.VsockSocket) if err != nil { log.WithFunc("pool.hibernateLocked").Warnf(ctx, "sync %s before hibernate, reads will wake it: %v", sb.ID, err) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index c88b4167..9e0124cb 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -150,7 +150,9 @@ type Engine interface { MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error SyncGuest(ctx context.Context, vsockSocket string) error - StartReader(ctx context.Context, vsockSocket string, d engine.ReaderDisks) error + DiskAttachSnapshot(ctx context.Context, vmName, snapName string) (string, error) + DiskAttachDir(ctx context.Context, vmName, dir string) (string, error) + StartReader(ctx context.Context, vsockSocket, cowDisk string) error DialReader(ctx context.Context, vsockSocket string) (net.Conn, error) WriteInstanceMetadata(ctx context.Context, vsockSocket string, doc []byte) error WriteGuestEnv(ctx context.Context, vsockSocket string, doc []byte) error @@ -462,6 +464,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg claimed: map[string]*types.Sandbox{}, pendingRemovals: map[string]pendingRemoval{}, tenantLive: map[string]int{}, + readers: &readerSet{live: map[string]*reader{}}, archiving: map[string]struct{}{}, pendingCks: map[string]struct{}{}, recLocks: map[string]*sync.RWMutex{}, @@ -474,9 +477,6 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg refillKick: make(chan struct{}, 1), healSem: make(chan struct{}, maxConcurrentHeals), } - if cfg.Reader != nil { - m.readers = &readerSet{key: cfg.Reader.Key(), live: map[string]*reader{}} - } for _, volume := range cfg.Volumes { m.volumes[volume.Name] = catalogVolume{ disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 083a5ea3..c24d783e 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -623,6 +623,7 @@ func TestSetPoolsRejectsConfigOwnedFields(t *testing.T) { {"capture_trim", config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}}, {"storage", config.PoolSpec{PoolKey: testKey, Warm: 1, Storage: "40G"}}, {"replicate_seconds", config.PoolSpec{PoolKey: testKey, Warm: 1, ReplicateSeconds: 60}}, + {"reader", config.PoolSpec{PoolKey: testKey, Warm: 1, Reader: true}}, } { err := m.SetPools(t.Context(), []config.PoolSpec{tt.spec}) if !errors.Is(err, ErrBadKey) || !strings.Contains(err.Error(), tt.field+" is set in the config file") { @@ -1221,12 +1222,14 @@ type fakeEngine struct { snapSaves, exports, snapshots []string exportContent []byte exportFiles map[string]string - startedReaders []engine.ReaderDisks + startedReaders []string + attachedCopies []string + attachedDirFiles []string + attachStall chan struct{} readerDials []string startReaderErr error readerDialErr error syncErr error - managedDirs []string caInstalls []string warmups [][]string warmupSocks []string @@ -1618,9 +1621,6 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.Volum if err := ctx.Err(); err != nil { return err } - if err := f.refuseManagedPath(spec.Path); err != nil { - return err - } f.mu.Lock() f.volumeSpecs = append(f.volumeSpecs, spec) f.volumeOps = append(f.volumeOps, "attach:"+spec.Name) @@ -1691,10 +1691,39 @@ func (f *fakeEngine) SyncGuest(_ context.Context, vsockSocket string) error { return f.syncErr } -func (f *fakeEngine) StartReader(_ context.Context, vsockSocket string, d engine.ReaderDisks) error { +func (f *fakeEngine) DiskAttachSnapshot(_ context.Context, vmName, snapName string) (string, error) { + f.mu.Lock() + f.attachedCopies = append(f.attachedCopies, "snapshot:"+vmName+":"+snapName) + stall, err := f.attachStall, f.diskAttachErr + f.mu.Unlock() + if stall != nil { + <-stall + } + if err != nil { + return "", err + } + return "snap-" + snapName, nil +} + +func (f *fakeEngine) DiskAttachDir(_ context.Context, vmName, dir string) (string, error) { + entries, err := os.ReadDir(dir) f.mu.Lock() defer f.mu.Unlock() - f.startedReaders = append(f.startedReaders, d) + if err = cmp.Or(err, f.diskAttachErr); err != nil { + return "", err + } + f.attachedCopies = append(f.attachedCopies, "dir:"+vmName) + f.attachedDirFiles = nil + for _, entry := range entries { + f.attachedDirFiles = append(f.attachedDirFiles, entry.Name()) + } + return "snap-dir", nil +} + +func (f *fakeEngine) StartReader(_ context.Context, vsockSocket, cowDisk string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.startedReaders = append(f.startedReaders, cowDisk) f.volumeOps = append(f.volumeOps, "reader-start:"+vsockSocket) return f.startReaderErr } @@ -1712,26 +1741,6 @@ func (f *fakeEngine) DialReader(_ context.Context, vsockSocket string) (net.Conn return conn, nil } -func (f *fakeEngine) refuseManagedPath(path string) error { - f.mu.Lock() - dirs := slices.Clone(f.managedDirs) - f.mu.Unlock() - resolved, err := filepath.EvalSymlinks(path) - if err != nil { - resolved = path - } - for _, dir := range dirs { - root, evalErr := filepath.EvalSymlinks(dir) - if evalErr != nil { - root = dir - } - if rel, relErr := filepath.Rel(root, resolved); relErr == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return fmt.Errorf("external volume path %s is inside a cocoon-managed directory", path) - } - } - return nil -} - func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index f677a689..c58c900c 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -117,7 +117,7 @@ func (m *Manager) adoptPools(ctx context.Context) error { return nil } -// poolSeedHash digests a pool set's warm-target shape, order-independent, without the config-owned egress, warmup, capture_trim, storage, replicate_seconds and egress_upstream_env. +// poolSeedHash digests a pool set's warm-target shape, order-independent, without the config-owned egress, warmup, capture_trim, storage, replicate_seconds, reader and egress_upstream_env. func poolSeedHash(specs []config.PoolSpec) string { shaped := slices.Clone(specs) for i := range shaped { @@ -126,6 +126,7 @@ func poolSeedHash(specs []config.PoolSpec) string { shaped[i].CaptureTrim = false shaped[i].Storage = "" shaped[i].ReplicateSeconds = 0 + shaped[i].Reader = false shaped[i].EgressUpstreamEnv = "" } slices.SortFunc(shaped, func(a, b config.PoolSpec) int { return a.Compare(b.PoolKey) }) diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index 8f89c598..5c3f38ea 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -88,6 +88,7 @@ func TestPoolSeedHashIgnoresConfigOwnedFields(t *testing.T) { "capture_trim": func(s *config.PoolSpec) { s.CaptureTrim = true }, "storage": func(s *config.PoolSpec) { s.Storage = "20G" }, "replicate_seconds": func(s *config.PoolSpec) { s.ReplicateSeconds = 60 }, + "reader": func(s *config.PoolSpec) { s.Reader = true }, "egress_upstream_env": func(s *config.PoolSpec) { s.EgressUpstreamEnv = "POOL_UPSTREAM" }, } { spec := base diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index fe839c2a..cfeb6970 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -15,31 +15,24 @@ import ( "github.com/projecteru2/core/log" "golang.org/x/sync/singleflight" - "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/types" - "github.com/cocoonstack/sandbox/sandboxd/utils" ) const ( readerIdle = 30 * time.Second readerSetupTimeout = 5 * time.Minute readerDir = "readers" - readerCowDisk = "rcow" - readerCowFile = "cow.raw" - readerSidecarFile = "cocoon.json" - layerRole = "layer" + envelopeFile = "snapshot.json" ) -var linkFile = os.Link - // DialReader opens a silkd connection to a sleeping claim's reader VM without waking the claim. func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Conn, source string, err error) { sb, ok := m.claim(id, token) if !ok { return nil, "", ErrUnknownSandbox } - if m.readers == nil { + if !m.view.Load().poolReaders[sb.Key] { return nil, "", ErrNoReader } r, err := m.acquireReader(ctx, sb) @@ -140,18 +133,18 @@ func (m *Manager) ensureReader(ctx context.Context, sb *types.Sandbox) error { func (m *Manager) openReader(ctx context.Context, sb *types.Sandbox) error { start := time.Now() r := &reader{claim: sb.ID, dir: filepath.Join(m.dataDir, readerDir, sb.ID+"-"+randHex(3))} - layers, err := m.readerSource(ctx, sb, r) + archived, err := m.stageReader(ctx, sb, r) if err != nil { _ = os.RemoveAll(r.dir) return err } - vm, err := m.takeReaderVM() + vm, err := m.takeReaderVM(sb.Key) if err != nil { _ = os.RemoveAll(r.dir) return err } r.vm, r.sock = vm.VMName, vm.VsockSocket - if err = m.startReader(ctx, r, layers); err != nil { + if err = m.startReader(ctx, sb, r, archived); err != nil { m.destroyReader(ctx, r) return err } @@ -168,62 +161,63 @@ func (m *Manager) openReader(ctx context.Context, sb *types.Sandbox) error { return nil } -func (m *Manager) readerSource(ctx context.Context, sb *types.Sandbox, r *reader) ([]string, error) { - archived, err := m.captureSleeping(ctx, sb, r) - if err != nil { - return nil, err - } - sidecar := filepath.Join(r.dir, readerSidecarFile) - if archived { - if sidecar, err = m.fetchReaderDisk(ctx, r); err != nil { - return nil, err - } +func (m *Manager) stageReader(ctx context.Context, sb *types.Sandbox, r *reader) (archived bool, err error) { + sb.Transition.Lock() + r.source, archived, err = m.readerSource(ctx, sb) + sb.Transition.Unlock() + if err != nil || !archived { + return archived, err } - return readerLayers(sidecar) + return true, m.fetchReaderArchive(ctx, r) } -// captureSleeping copies a hibernated snapshot under the Transition lock, which the wake that consumes the snapshot takes. -func (m *Manager) captureSleeping(ctx context.Context, sb *types.Sandbox, r *reader) (archived bool, err error) { - sb.Transition.Lock() - defer sb.Transition.Unlock() +// readerSource names the sleep a reader would serve; callers hold sb.Transition, which the wake that consumes the sleep takes. +func (m *Manager) readerSource(ctx context.Context, sb *types.Sandbox) (source string, archived bool, err error) { if (sb.HibernateSnap == "" && sb.ArchiveCk == "") || !sb.ReaderSafe { - return false, errNotSleeping + return "", false, errNotSleeping } - if sb.HibernateSnap == "" { + archived = sb.HibernateSnap == "" + if archived { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { - return false, err + return "", false, err } } if err = m.diskErr(); err != nil { - return false, err + return "", false, err } - if sb.HibernateSnap == "" { - r.source = sb.ArchiveCk - return true, nil - } - r.source = sb.HibernateSnap - if err = m.eng.SnapshotExport(ctx, r.source, r.dir); err != nil { - return false, fmt.Errorf("export snapshot: %w", err) - } - return false, utils.RemoveDirEntries(r.dir, func(name string) bool { return name != readerCowFile && name != readerSidecarFile }) + return cmp.Or(sb.HibernateSnap, sb.ArchiveCk), archived, nil } -func (m *Manager) fetchReaderDisk(ctx context.Context, r *reader) (string, error) { +func (m *Manager) fetchReaderArchive(ctx context.Context, r *reader) error { if err := os.MkdirAll(r.dir, 0o750); err != nil { - return "", err + return err + } + if err := m.fetchReaderFile(ctx, r, envelopeFile); err != nil { + return err } - for _, name := range []string{readerCowFile, readerSidecarFile} { - if err := m.ckpts.FetchFile(ctx, r.source, name, filepath.Join(r.dir, name)); err != nil { - return "", archiveFetchErr(err) + names, err := diskAttachFiles(filepath.Join(r.dir, envelopeFile)) + if err != nil { + return err + } + for _, name := range names { + if err = m.fetchReaderFile(ctx, r, name); err != nil { + return err } } - return filepath.Join(r.dir, readerSidecarFile), nil + return nil +} + +func (m *Manager) fetchReaderFile(ctx context.Context, r *reader, name string) error { + if err := m.ckpts.FetchFile(ctx, r.source, name, filepath.Join(r.dir, name)); err != nil { + return archiveFetchErr(err) + } + return nil } -func (m *Manager) takeReaderVM() (*types.Sandbox, error) { +func (m *Manager) takeReaderVM(key types.PoolKey) (*types.Sandbox, error) { m.mu.Lock() var sb *types.Sandbox - if p, ok := m.activePool(m.readers.key); ok { + if p, ok := m.activePool(key); ok { sb = p.popWarm() } m.mu.Unlock() @@ -234,25 +228,25 @@ func (m *Manager) takeReaderVM() (*types.Sandbox, error) { return sb, nil } -func (m *Manager) startReader(ctx context.Context, r *reader, layers []string) error { - disks := engine.ReaderDisks{Cow: readerCowDisk} - for i, blob := range layers { - name := fmt.Sprintf("rl%d", i) - linked := filepath.Join(r.dir, name+".erofs") - if err := linkFile(blob, linked); err != nil { - if err = utils.CloneFile(blob, linked); err != nil { - return fmt.Errorf("link layer %d into the reader dir: %w", i, err) - } - } - if err := m.eng.DiskAttach(ctx, r.vm, engine.VolumeSpec{Name: name, Path: linked}); err != nil { - return fmt.Errorf("attach layer %d: %w", i, err) - } - disks.Layers = append(disks.Layers, name) - } - if err := m.eng.DiskAttach(ctx, r.vm, engine.VolumeSpec{Name: readerCowDisk, Path: filepath.Join(r.dir, readerCowFile), RW: true}); err != nil { +func (m *Manager) startReader(ctx context.Context, sb *types.Sandbox, r *reader, archived bool) error { + disk, err := m.attachReaderDisk(ctx, sb, r, archived) + if err != nil { return fmt.Errorf("attach claim disk: %w", err) } - return m.eng.StartReader(ctx, r.sock, disks) + return m.eng.StartReader(ctx, r.sock, disk) +} + +func (m *Manager) attachReaderDisk(ctx context.Context, sb *types.Sandbox, r *reader, archived bool) (string, error) { + if archived { + defer func() { _ = os.RemoveAll(r.dir) }() + return m.eng.DiskAttachDir(ctx, r.vm, r.dir) + } + sb.Transition.Lock() + defer sb.Transition.Unlock() + if source, _, err := m.readerSource(ctx, sb); err != nil || source != r.source { + return "", cmp.Or(err, errNotSleeping) + } + return m.eng.DiskAttachSnapshot(ctx, r.vm, r.source) } func (m *Manager) destroyReader(ctx context.Context, r *reader) { @@ -263,7 +257,6 @@ func (m *Manager) destroyReader(ctx context.Context, r *reader) { } type readerSet struct { - key types.PoolKey mu sync.Mutex live map[string]*reader flights singleflight.Group @@ -294,32 +287,24 @@ func archiveFetchErr(err error) error { return fmt.Errorf("fetch archive: %w", err) } -func readerLayers(sidecar string) ([]string, error) { - raw, err := os.ReadFile(sidecar) //nolint:gosec // path from our own export or fetch +func diskAttachFiles(envelope string) ([]string, error) { + raw, err := os.ReadFile(envelope) //nolint:gosec // path from our own fetch if err != nil { - return nil, fmt.Errorf("read snapshot sidecar: %w", err) + return nil, fmt.Errorf("read archive envelope: %w", err) } - var meta struct { - StorageConfigs []struct { - Path string `json:"path"` - Role string `json:"role"` - } `json:"storage_configs"` + var doc struct { + Files []string `json:"disk_attach_files"` } - if err = json.Unmarshal(raw, &meta); err != nil { - return nil, fmt.Errorf("parse snapshot sidecar: %w", err) + if err = json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("parse archive envelope: %w", err) } - var layers []string - for _, sc := range meta.StorageConfigs { - if sc.Role != layerRole { - continue - } - if _, err = os.Stat(sc.Path); err != nil { - return nil, fmt.Errorf("layer blob: %w", err) - } - layers = append(layers, sc.Path) + if len(doc.Files) == 0 { + return nil, errors.New("archive envelope lists no disk_attach_files") } - if len(layers) == 0 { - return nil, errors.New("snapshot sidecar names no layers") + for _, name := range doc.Files { + if filepath.Base(name) != name || !filepath.IsLocal(name) { + return nil, fmt.Errorf("archive envelope lists disk attach file %q outside its directory", name) + } } - return layers, nil + return doc.Files, nil } diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 39dc5d4b..e7961033 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -7,28 +7,23 @@ import ( "os" "path/filepath" "slices" - "strings" "sync" "testing" "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/types" ) -var readerKey = types.PoolKey{Template: "reader:24.04", Net: types.NetNone, Size: types.SizeSmall} - func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { eng := newFakeEngine() - layers := readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { t.Fatalf("WakeAgentSocketOrRead = %v, want ErrSleeping", err) @@ -48,104 +43,117 @@ func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { if len(eng.restores) != 0 || sb.HibernateSnap == "" { t.Errorf("claim woke: restores=%v hibernate snapshot=%q", eng.restores, sb.HibernateSnap) } - if got, want := readerOps(eng), []string{"attach:rl0", "attach:rl1", "attach:rcow", "reader-start:/vsock/sbx-reader-1"}; !slices.Equal(got, want) { - t.Errorf("reader setup ran %v, want %v", got, want) - } - cow := eng.volumeSpecs[len(eng.volumeSpecs)-1] - dir := filepath.Dir(cow.Path) - wantSpecs := []engine.VolumeSpec{{Name: "rl0", Path: filepath.Join(dir, "rl0.erofs")}, {Name: "rl1", Path: filepath.Join(dir, "rl1.erofs")}, {Name: "rcow", Path: cow.Path, RW: true}} - if !slices.Equal(eng.volumeSpecs, wantSpecs) { - t.Errorf("attached %+v, want read-only layers linked into the reader dir, then the writable copy %+v", eng.volumeSpecs, wantSpecs) - } - for i, blob := range layers { - linked, statErr := os.Stat(wantSpecs[i].Path) - orig, origErr := os.Stat(blob) - if statErr != nil || origErr != nil || !os.SameFile(linked, orig) { - t.Errorf("layer %d attached as %s, want a hard link of %s: %v %v", i, wantSpecs[i].Path, blob, statErr, origErr) - } - } - if want := (engine.ReaderDisks{Layers: []string{"rl0", "rl1"}, Cow: "rcow"}); len(eng.startedReaders) != 1 || !slices.Equal(eng.startedReaders[0].Layers, want.Layers) || eng.startedReaders[0].Cow != want.Cow { - t.Errorf("view started with %+v, want %+v", eng.startedReaders, want) + if want := []string{"snapshot:sbx-reader-1:" + sb.HibernateSnap}; !slices.Equal(eng.attachedCopies, want) { + t.Errorf("attached %v, want the snapshot copied onto the reader VM %v", eng.attachedCopies, want) } - if got, readErr := os.ReadFile(cow.Path); readErr != nil || string(got) != "disk-bytes" { - t.Errorf("reader disk %q, %v, want the snapshot's copy-on-write bytes", got, readErr) + if want := []string{"snap-" + sb.HibernateSnap}; !slices.Equal(eng.startedReaders, want) { + t.Errorf("view started over %v, want the serial the attach returned %v", eng.startedReaders, want) } - entries, _ := os.ReadDir(dir) - names := make([]string, len(entries)) - for i, entry := range entries { - names[i] = entry.Name() + if n := len(m.pools[testKey].warm); n != 0 { + t.Errorf("claim pool holds %d warm VMs, want the one reader taken", n) } - if want := []string{"cocoon.json", "cow.raw", "rl0.erofs", "rl1.erofs"}; !slices.Equal(names, want) { - t.Errorf("reader dir holds %v, want only %v", names, want) - } - if n := len(m.pools[readerKey].warm); n != 0 { - t.Errorf("reader pool holds %d warm VMs, want the one reader taken", n) + if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { + t.Errorf("a hibernated read left %d entries under %s", len(entries), readerDir) } } -func TestReaderLayersFallBackToACopyWhenTheLinkFails(t *testing.T) { - linkFile = func(string, string) error { return errors.New("cross-device link") } - defer func() { linkFile = os.Link }() +func TestReaderTakesItsVMFromTheClaimsOwnPool(t *testing.T) { eng := newFakeEngine() - layers := readerExport(t, eng) - m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") + m := newReaderManager(t, eng, config.PoolSpec{PoolKey: otherKey, Warm: 1}) sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-own") + seedReaderVMs(m, otherKey, "sbx-foreign") conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("DialReader: %v", err) } _ = conn.Close() - for i, blob := range layers { - attached := eng.volumeSpecs[i].Path - copied, copyErr := os.Stat(attached) - orig, origErr := os.Stat(blob) - if copyErr != nil || origErr != nil || os.SameFile(copied, orig) { - t.Errorf("layer %d attached as %s, want a copy distinct from %s: %v %v", i, attached, blob, copyErr, origErr) - } - if got, _ := os.ReadFile(attached); string(got) != "erofs" { - t.Errorf("layer %d copy holds %q, want the blob's bytes", i, got) - } + if want := []string{"snapshot:sbx-own:" + sb.HibernateSnap}; !slices.Equal(eng.attachedCopies, want) { + t.Errorf("attached %v, want the claim's own pool VM %v", eng.attachedCopies, want) + } + if n := len(m.pools[otherKey].warm); n != 1 { + t.Errorf("another pool holds %d warm VMs, want it untouched", n) } } -func TestReaderReleaseRemovesItsLayerLinksAndKeepsTheBlobs(t *testing.T) { +func TestReaderServesOnlyPoolsThatAskForIt(t *testing.T) { eng := newFakeEngine() - layers := readerExport(t, eng) - m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") - sb := mustClaim(t, m, testKey) + m := newReaderManager(t, eng, config.PoolSpec{PoolKey: otherKey, Warm: 1}) + sb := mustClaim(t, m, otherKey) hibernateClaim(t, m, sb) - conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + seedReaderVMs(m, otherKey, "sbx-reader-1") + + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Errorf("DialReader of a pool without reader = %v, want ErrNoReader", err) + } + sock, _, done, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + if err != nil || sock == "" { + t.Fatalf("WakeAgentSocketOrRead = %q, %v, want a wake", sock, err) + } + defer done() + if len(eng.restores) != 1 || len(eng.attachedCopies) != 0 || len(m.pools[otherKey].warm) != 1 { + t.Errorf("restores=%v attached=%v warm=%d, want a wake with no reader taken", eng.restores, eng.attachedCopies, len(m.pools[otherKey].warm)) + } +} + +func TestReloadTurnsTheReaderOnForLaterSleeps(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + next := *m.cfg + next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Reader: true}} + res, err := m.ReloadConfig(t.Context(), &next) if err != nil { - t.Fatalf("DialReader: %v", err) + t.Fatalf("reload: %v", err) } - _ = conn.Close() + if want := []string{"pools[rt:24.04 none small].reader"}; !slices.Equal(res.Changed, want) { + t.Errorf("changed %v, want %v", res.Changed, want) + } + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + if _, _, _, err = m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + t.Errorf("WakeAgentSocketOrRead after the reload = %v, want ErrSleeping", err) + } +} - m.readers.mu.Lock() - live := m.readers.live[sb.ID] - m.readers.mu.Unlock() - m.dropReader(t.Context(), live, true) - for _, blob := range layers { - if got, readErr := os.ReadFile(blob); readErr != nil || string(got) != "erofs" { - t.Errorf("release damaged the shared blob %s: %q, %v", blob, got, readErr) +func TestReaderHoldsTheTransitionLockWhileTheSnapshotIsCopied(t *testing.T) { + eng := newFakeEngine() + eng.attachStall = make(chan struct{}) + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") + + var wg sync.WaitGroup + wg.Go(func() { + conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Errorf("DialReader: %v", err) + return } + _ = conn.Close() + }) + waitFor(t, func() bool { eng.mu.Lock(); defer eng.mu.Unlock(); return len(eng.attachedCopies) > 0 }) + if sb.Transition.TryLock() { + sb.Transition.Unlock() + t.Error("the transition lock was free while the snapshot was being copied") } - if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { - t.Errorf("release left %d entries under %s", len(entries), readerDir) + close(eng.attachStall) + wg.Wait() + if !sb.Transition.TryLock() { + t.Fatal("the reader kept the transition lock after the copy") } + sb.Transition.Unlock() } func TestReaderCallsShareOneSetup(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) - eng.exportStall = make(chan struct{}) + eng.attachStall = make(chan struct{}) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1", "sbx-reader-2") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1", "sbx-reader-2") const callers = 8 var wg sync.WaitGroup @@ -159,14 +167,14 @@ func TestReaderCallsShareOneSetup(t *testing.T) { _ = conn.Close() }) } - waitFor(t, func() bool { eng.mu.Lock(); defer eng.mu.Unlock(); return len(eng.exports) > 0 }) - close(eng.exportStall) + waitFor(t, func() bool { eng.mu.Lock(); defer eng.mu.Unlock(); return len(eng.attachedCopies) > 0 }) + close(eng.attachStall) wg.Wait() eng.mu.Lock() defer eng.mu.Unlock() - if len(eng.exports) != 1 || len(eng.startedReaders) != 1 { - t.Errorf("%d callers ran %d exports and %d reader setups, want one of each", callers, len(eng.exports), len(eng.startedReaders)) + if len(eng.attachedCopies) != 1 || len(eng.startedReaders) != 1 { + t.Errorf("%d callers ran %d attaches and %d reader setups, want one of each", callers, len(eng.attachedCopies), len(eng.startedReaders)) } if len(eng.readerDials) != callers { t.Errorf("%d dials, want one per caller", len(eng.readerDials)) @@ -176,11 +184,10 @@ func TestReaderCallsShareOneSetup(t *testing.T) { func TestReaderIsReleasedAfterItsIdleWindow(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") dialAndClose := func() { t.Helper() conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) @@ -190,7 +197,6 @@ func TestReaderIsReleasedAfterItsIdleWindow(t *testing.T) { _ = conn.Close() } dialAndClose() - dir := filepath.Dir(eng.volumeSpecs[len(eng.volumeSpecs)-1].Path) time.Sleep(readerIdle * 2 / 3) dialAndClose() @@ -205,9 +211,6 @@ func TestReaderIsReleasedAfterItsIdleWindow(t *testing.T) { if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { t.Errorf("removes=%v, want the idle reader's VM destroyed", removes) } - if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { - t.Errorf("reader dir survived the release: %v", err) - } m.readers.mu.Lock() defer m.readers.mu.Unlock() if len(m.readers.live) != 0 { @@ -219,11 +222,10 @@ func TestReaderIsReleasedAfterItsIdleWindow(t *testing.T) { func TestReaderNoConnectionAcquiresIsReleasedAfterTheIdleWindow(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") if err := m.ensureReader(t.Context(), sb); err != nil { t.Fatalf("ensureReader: %v", err) } @@ -243,11 +245,10 @@ func TestReaderNoConnectionAcquiresIsReleasedAfterTheIdleWindow(t *testing.T) { func TestReaderRegistrationRetiresTheOneItReplaces(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-2", "sbx-reader-1") for range 2 { if err := m.openReader(t.Context(), sb); err != nil { @@ -267,11 +268,10 @@ func TestReaderRegistrationRetiresTheOneItReplaces(t *testing.T) { func TestReaderEnsureSkipsASetupAnotherFlightFinished(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-2", "sbx-reader-1") for range 2 { if err := m.ensureReader(t.Context(), sb); err != nil { @@ -285,11 +285,10 @@ func TestReaderEnsureSkipsASetupAnotherFlightFinished(t *testing.T) { func TestReaderKeepsTheClaimsLatestSleepOnly(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-2", "sbx-reader-1") first, firstSource, err := m.DialReader(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("DialReader: %v", err) @@ -314,23 +313,23 @@ func TestReaderKeepsTheClaimsLatestSleepOnly(t *testing.T) { if secondSource == firstSource { t.Errorf("both sleeps served from %q", firstSource) } - if len(eng.exports) != 2 || len(eng.startedReaders) != 2 { - t.Errorf("exports=%d setups=%d, want a fresh reader per sleep", len(eng.exports), len(eng.startedReaders)) + if len(eng.attachedCopies) != 2 || len(eng.startedReaders) != 2 { + t.Errorf("attaches=%d setups=%d, want a fresh reader per sleep", len(eng.attachedCopies), len(eng.startedReaders)) } if !slices.Contains(eng.removes, "sbx-reader-1") { t.Errorf("removes=%v, want the first sleep's reader destroyed", eng.removes) } } -func TestReaderServesAnArchivedClaimFromTwoFilesOfItsRecord(t *testing.T) { +func TestReaderServesAnArchivedClaimFromTheFilesItsEnvelopeLists(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) + readerExport(eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") counting := &fetchCountingStore{Store: m.ckpts} m.ckpts = counting sb := mustClaim(t, m, testKey) mustArchive(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") clones := len(eng.clones) + len(eng.colds) if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { @@ -348,53 +347,75 @@ func TestReaderServesAnArchivedClaimFromTwoFilesOfItsRecord(t *testing.T) { if ck == "" || source != ck { t.Errorf("reader source %q, want the archive record %q", source, ck) } - if got, want := counting.files(), []string{"cocoon.json", "cow.raw"}; !slices.Equal(got, want) { + if got, want := counting.files(), []string{"disk.img", envelopeFile}; !slices.Equal(got, want) { t.Errorf("fetched files %v, want only %v", got, want) } if counting.fetches != 0 { t.Errorf("%d whole-record fetches, want none", counting.fetches) } + if want := []string{"disk.img", envelopeFile}; !slices.Equal(eng.attachedDirFiles, want) { + t.Errorf("attached directory held %v, want the envelope and its listed file %v", eng.attachedDirFiles, want) + } + if want := []string{"dir:sbx-reader-1"}; !slices.Equal(eng.attachedCopies, want) { + t.Errorf("attached %v, want the directory copied onto the reader VM %v", eng.attachedCopies, want) + } if n := len(eng.clones) + len(eng.colds); n != clones || len(eng.restores) != 0 { t.Errorf("an archived read booted a VM: clones/colds %d->%d, restores %v", clones, n, eng.restores) } if !ckExists(t, m, ck) { t.Error("the archive record was consumed") } - cow := eng.volumeSpecs[len(eng.volumeSpecs)-1] - if got, readErr := os.ReadFile(cow.Path); readErr != nil || string(got) != "disk-bytes" { - t.Errorf("reader disk %q, %v, want the record's bytes", got, readErr) - } - if !strings.HasPrefix(cow.Path, filepath.Join(m.dataDir, readerDir)) { - t.Errorf("reader attached %s, want a private copy under %s", cow.Path, readerDir) + if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { + t.Errorf("the fetched copy outlived its attach: %d entries under %s", len(entries), readerDir) } +} - m.readers.mu.Lock() - live := m.readers.live[sb.ID] - m.readers.mu.Unlock() - m.dropReader(t.Context(), live, true) - if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { - t.Errorf("a released archived reader left %d entries under %s", len(entries), readerDir) +func TestReaderRefusesAnArchiveWithoutUsableAttachFiles(t *testing.T) { + for _, tt := range []struct{ name, envelope string }{ + {"older export", `{"version":1}`}, + {"empty list", `{"version":1,"disk_attach_files":[]}`}, + {"parent path", `{"version":1,"disk_attach_files":["../escape"]}`}, + {"nested path", `{"version":1,"disk_attach_files":["sub/disk.img"]}`}, + {"absolute path", `{"version":1,"disk_attach_files":["/etc/passwd"]}`}, + } { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + eng.exportFiles = map[string]string{envelopeFile: tt.envelope, "disk.img": "disk-bytes"} + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + mustArchive(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") + + if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { + t.Fatalf("DialReader = %v, want ErrNoReader", err) + } + if len(m.pools[testKey].warm) != 1 || len(eng.attachedCopies) != 0 { + t.Errorf("a refused archive took a reader: warm=%d attached=%v", len(m.pools[testKey].warm), eng.attachedCopies) + } + if entries, _ := os.ReadDir(filepath.Join(m.dataDir, readerDir)); len(entries) != 0 { + t.Errorf("a refused archive left %d entries under %s", len(entries), readerDir) + } + }) } } func TestReaderRefusesWhatAWakeRefuses(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) - cfg := &config.Config{ - DataDir: t.TempDir(), APIToken: "root", Reader: &config.ReaderSpec{Template: readerKey.Template}, - Pools: []config.PoolSpec{archivePool(0), {PoolKey: readerKey, Warm: 1}}, - } + readerExport(eng) + pool := archivePool(0) + pool.Reader = true + cfg := &config.Config{DataDir: t.TempDir(), APIToken: "root", Pools: []config.PoolSpec{pool}} m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) if err != nil { t.Fatalf("setup manager: %v", err) } - seedReaderVMs(m, "sbx-reader-1") putTenants(t, m, tenants.TenantSpec{Name: "acme", Token: "acme-tok"}) sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "acme"}) if err != nil { t.Fatalf("claim: %v", err) } mustArchive(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") if err = m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatalf("DeleteTenant: %v", err) } @@ -402,25 +423,24 @@ func TestReaderRefusesWhatAWakeRefuses(t *testing.T) { if _, _, err = m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) || !errors.Is(err, ErrTenantRemoved) { t.Errorf("DialReader of a removed tenant's archive = %v, want ErrNoReader wrapping ErrTenantRemoved", err) } - if len(eng.startedReaders) != 0 || len(m.pools[readerKey].warm) != 1 { - t.Errorf("a refused read still took a reader: setups=%d warm=%d", len(eng.startedReaders), len(m.pools[readerKey].warm)) + if len(eng.startedReaders) != 0 || len(m.pools[testKey].warm) != 1 { + t.Errorf("a refused read still took a reader: setups=%d warm=%d", len(eng.startedReaders), len(m.pools[testKey].warm)) } } func TestReaderRefusesWhileTheDiskIsLow(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") m.diskLow.Store(true) if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) || !errors.Is(err, ErrQuota) { t.Errorf("DialReader on a full disk = %v, want ErrNoReader wrapping ErrQuota", err) } - if len(eng.exports) != 0 || len(m.pools[readerKey].warm) != 1 { - t.Errorf("a refused read still exported (%d) or took a reader (%d warm)", len(eng.exports), len(m.pools[readerKey].warm)) + if len(eng.attachedCopies) != 0 || len(m.pools[testKey].warm) != 1 { + t.Errorf("a refused read still attached (%d) or took a reader (%d warm)", len(eng.attachedCopies), len(m.pools[testKey].warm)) } } @@ -430,18 +450,17 @@ func TestReaderSetupFailureFallsBackAndCleansUp(t *testing.T) { setup func(*fakeEngine) warm []string }{ - {"attach fails", func(e *fakeEngine) { e.diskAttachErrFor = "rcow" }, []string{"sbx-reader-1"}}, + {"attach fails", func(e *fakeEngine) { e.diskAttachErr = errors.New("attach failed") }, []string{"sbx-reader-1"}}, {"view fails", func(e *fakeEngine) { e.startReaderErr = errors.New("mount failed") }, []string{"sbx-reader-1"}}, {"no warm reader", func(*fakeEngine) {}, nil}, } { t.Run(tt.name, func(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) tt.setup(eng) m := newReaderManager(t, eng) - seedReaderVMs(m, tt.warm...) sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, tt.warm...) if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { t.Fatalf("DialReader = %v, want ErrNoReader", err) @@ -492,11 +511,10 @@ func TestSleepingSourceNamesOnlyASettledSleepOfTheCallersClaim(t *testing.T) { func TestReaderThatStopsAnsweringIsRetired(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-2", "sbx-reader-1") eng.readerDialErr = errors.New("vsock refused") if _, _, err := m.DialReader(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrNoReader) { @@ -548,11 +566,10 @@ func TestWakeAgentSocketStampsActivityOnlyWhenItServesTheGuest(t *testing.T) { func TestReaderOfAStaleSleepIsRetiredEvenWithAConnectionOpen(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-2", "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-2", "sbx-reader-1") open, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("DialReader: %v", err) @@ -584,12 +601,11 @@ func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { eng := newFakeEngine() - readerExport(t, eng) eng.syncErr = tt.syncErr m := newReaderManager(t, eng) - seedReaderVMs(m, "sbx-reader-1") sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") if got, _ := newClaimStore(m.dataDir, false).load(); got[sb.ID] == nil || got[sb.ID].ReaderSafe != tt.safe { t.Errorf("persisted reader_safe = %+v, want %t", got[sb.ID], tt.safe) @@ -720,13 +736,9 @@ func TestReconcileClearsReaderLeftovers(t *testing.T) { } } -func newReaderManager(t *testing.T, eng *fakeEngine) *Manager { +func newReaderManager(t *testing.T, eng *fakeEngine, extra ...config.PoolSpec) *Manager { t.Helper() - cfg := &config.Config{ - DataDir: t.TempDir(), - Reader: &config.ReaderSpec{Template: readerKey.Template}, - Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}, {PoolKey: readerKey, Warm: 1}}, - } + cfg := &config.Config{DataDir: t.TempDir(), Pools: append([]config.PoolSpec{{PoolKey: testKey, Warm: 1, Reader: true}}, extra...)} m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) if err != nil { t.Fatalf("setup manager: %v", err) @@ -734,9 +746,9 @@ func newReaderManager(t *testing.T, eng *fakeEngine) *Manager { return m } -func seedReaderVMs(m *Manager, names ...string) { +func seedReaderVMs(m *Manager, key types.PoolKey, names ...string) { for _, name := range names { - m.pools[readerKey].warm = append(m.pools[readerKey].warm, &types.Sandbox{VMName: name, Key: readerKey, VsockSocket: "/vsock/" + name}) + m.pools[key].warm = append(m.pools[key].warm, &types.Sandbox{VMName: name, Key: key, VsockSocket: "/vsock/" + name}) } } @@ -747,28 +759,13 @@ func hibernateClaim(t *testing.T, m *Manager, sb *types.Sandbox) { } } -// readerExport makes every snapshot export carry the disk, a sidecar naming two layers, and a memory image a reader must drop. -func readerExport(t *testing.T, eng *fakeEngine) []string { - t.Helper() - dir := t.TempDir() - layers := []string{filepath.Join(dir, "base.erofs"), filepath.Join(dir, "top.erofs")} - for _, layer := range layers { - if err := os.WriteFile(layer, []byte("erofs"), 0o600); err != nil { - t.Fatalf("setup: %v", err) - } +// readerExport makes every snapshot export carry an envelope that lists one disk file for a reader to attach, and a memory image it must not fetch. +func readerExport(eng *fakeEngine) { + eng.exportFiles = map[string]string{ + envelopeFile: `{"version":1,"disk_attach_files":["disk.img"]}`, + "disk.img": "disk-bytes", + "memory-ranges": "guest-memory", } - sidecar := fmt.Sprintf(`{"storage_configs":[{"path":%q,"role":"layer"},{"path":%q,"role":"layer"},{"path":"/run/cow.raw","role":"cow"},{"path":"/run/cidata.img","role":"cidata"}]}`, layers[0], layers[1]) - eng.exportFiles = map[string]string{"cow.raw": "disk-bytes", "cocoon.json": sidecar, "memory-ranges": "guest-memory"} - eng.managedDirs = []string{dir} - return layers -} - -func readerOps(eng *fakeEngine) []string { - eng.mu.Lock() - defer eng.mu.Unlock() - return slices.DeleteFunc(slices.Clone(eng.volumeOps), func(op string) bool { - return !strings.HasPrefix(op, "attach:") && !strings.HasPrefix(op, "reader-start:") - }) } func readerLifecycle(eng *fakeEngine) (removes []string, setups int) { diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index f994cdd6..442a3aac 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -50,6 +50,9 @@ func (m *Manager) desiredPools(specs []config.PoolSpec) (map[types.PoolKey]confi if spec.CaptureTrim { return nil, fmt.Errorf("%w: pool %q: capture_trim is set in the config file, not via the API", ErrBadKey, spec.Template) } + if spec.Reader { + return nil, fmt.Errorf("%w: pool %q: reader is set in the config file, not via the API", ErrBadKey, spec.Template) + } if spec.Storage != "" { return nil, fmt.Errorf("%w: pool %q: storage is set in the config file, not via the API", ErrBadKey, spec.Template) } diff --git a/sandboxd/pool/view.go b/sandboxd/pool/view.go index 1b904bef..9e26058f 100644 --- a/sandboxd/pool/view.go +++ b/sandboxd/pool/view.go @@ -16,6 +16,7 @@ type configView struct { poolTrims map[types.PoolKey]bool poolStorage map[types.PoolKey]string poolReplicate map[types.PoolKey]time.Duration + poolReaders map[types.PoolKey]bool diskReserve int64 diskDirs []string } @@ -27,6 +28,7 @@ func newConfigView(cfg *config.Config, secrets *egress.SecretStore) *configView poolTrims: map[types.PoolKey]bool{}, poolStorage: map[types.PoolKey]string{}, poolReplicate: map[types.PoolKey]time.Duration{}, + poolReaders: map[types.PoolKey]bool{}, } if r := cfg.DiskReserve; r != nil { v.diskReserve, _ = config.StorageBytes(r.Free) @@ -49,6 +51,9 @@ func newConfigView(cfg *config.Config, secrets *egress.SecretStore) *configView if spec.ReplicateSeconds > 0 { v.poolReplicate[key] = time.Duration(spec.ReplicateSeconds) * time.Second } + if spec.Reader { + v.poolReaders[key] = true + } } return v } diff --git a/sandboxd/pool/view_test.go b/sandboxd/pool/view_test.go index 3d275fef..d97a5101 100644 --- a/sandboxd/pool/view_test.go +++ b/sandboxd/pool/view_test.go @@ -16,6 +16,7 @@ func editView(m *Manager, edit func(v *configView)) { next.poolWarmups = maps.Clone(cur.poolWarmups) next.poolTrims = maps.Clone(cur.poolTrims) next.poolStorage = maps.Clone(cur.poolStorage) + next.poolReaders = maps.Clone(cur.poolReaders) edit(&next) m.view.Store(&next) } From a9829db42016a7fe4babd5e5267e1072e557f0c8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 12:54:16 +0800 Subject: [PATCH 20/28] docs: reader is a per-pool flag served through cocoon's disk attach --- README.md | 4 +- docs/deploy.md | 115 ++++++++++++++++++++++--------------------- docs/sandboxd-api.md | 14 +++--- docs/sdk.md | 2 +- 4 files changed, 70 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 13361755..d8223efb 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ performance) — source in promote/checkpoint HTTP API, operator-catalog dataset volumes (read-only or writable; read-only ones change on a running claim), signed preview URLs, the HTTP-upgrade byte relays to silkd and to any guest port, read-only file - access to hibernated or archived claims from a disposable reader VM - (opt-in `reader`), claim env with + access to hibernated or archived claims from a warm VM of their pool + (opt-in per pool `reader`), claim env with host-side credential injection and per-claim upstream proxies, runtime tenants, config reload (SIGHUP), usage + audit journals, /metrics, reap + restart reconcile with VMM cold restart, memberlist mesh with redirect diff --git a/docs/deploy.md b/docs/deploy.md index 3ef556a5..5d7d66d5 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -141,6 +141,7 @@ gets the intersection of the two. | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence. The pool follows the target down: warm VMs above it are destroyed once the count has stayed above it for a full decay period, so a burst does not leave the pool parked at its high-water mark | | `warmup` (pool entry) | unset | argv run in the golden VM after readiness and before its snapshot, so the files it touches are page-cache-resident in every clone, and again in every clone before it joins the warm pool, so those pages are already faulted into the restored VM when the first command runs — e.g. `["node", "-e", "0"]` on a Node flavor. It runs under the engine's 2-minute command timeout in silkd's base environment (`PATH`, `TERM`, and the guest image's proxy variables wherever nothing routes directly — the none lane and the locked bridge egress lane — with the proxy not yet serving, since a door pre-bound at refill only starts serving at claim); a non-zero exit or a timeout fails the golden build, so the pool stays unfilled until the config is fixed. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a golden built with a different warmup is rebuilt | | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | +| `reader` (pool entry) | `false` | Serve read-only file calls on this pool's hibernated or archived claims from a warm VM of the same pool instead of waking them; needs `warm` of at least 1 and a cocoon whose `vm disk attach` takes `--from-snapshot` and `--from-dir`. See [Reading sleeping claims](#reading-sleeping-claims-without-waking-them). Config-owned like `capture_trim`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one more pass, since a claim can run first in one pass and last in the next. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory; the first round of a claim after a wake or adoption uploads the whole claim, so that pass is the long pass. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | @@ -148,7 +149,6 @@ gets the intersection of the two. | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no warm VMs on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | -| `reader` | unset (off) | `{"template": "reader:24.04", "size": "small"}`: serve read-only file calls on a hibernated or archived claim from a disposable reader VM instead of waking it; the named none-lane pool must also be a `pools[]` entry with `warm` of at least 1. See [Reading sleeping claims](#reading-sleeping-claims-without-waking-them). Restart only | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); a lease moves only when a client calls [`/renew`](sandboxd-api.md#post-v1sandboxesidrenew). Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way, unless it claimed `on_expire: archive`, which archives at the lease end whatever this setting. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived; the claim is then gone for good. On S3, chunks and manifests remain until [offline reclamation](#reclaiming-s3-storage). On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. Same node-wide/per-pool split | @@ -172,70 +172,75 @@ fragment the warm pools): A client that only lists or reads files (a UI showing an idle agent's workspace, a tool collecting results) would otherwise wake the claim and hold its memory until the next idle hibernate, and for an archived claim -fetch the whole checkpoint first. With `reader` set, `fs_stat`, `fs_list`, -`fs_read` and `fs_pull` on a hibernated or archived claim are served from a -reader instead; anything else on the connection wakes the claim as before -([API](sandboxd-api.md#get-v1sandboxesidagent)). +fetch the whole checkpoint first. With `reader` set on the claim's pool, +`fs_stat`, `fs_list`, `fs_read` and `fs_pull` on a hibernated or archived +claim are served from a reader instead; anything else on the connection wakes +the claim as before ([API](sandboxd-api.md#get-v1sandboxesidagent)). ```json { - "reader": {"template": "reader:24.04", "size": "small"}, "pools": [ - {"template": "reader:24.04", "size": "small", "warm": 2}, - {"template": "rt:24.04", "idle_hibernate_seconds": 600} + {"template": "rt:24.04", "warm": 4, "reader": true, "idle_hibernate_seconds": 600} ] } ``` -The reader pool is an ordinary `pools[]` entry on the none lane, so it is -warmed, refilled and tuned (`PUT /v1/pools`) like any other. Taking a reader -for a claim removes one VM from it until the refill replaces it. A reader is -never shared: it serves one claim, and only the claim's own token reaches it. - -The guest's own kernel does the parsing, as it does for the claim. A reader -is built per claim from the claim's hibernate snapshot (or archive record): - -- the claim's layer blobs, read from the snapshot's sidecar, are hard-linked - into the reader's directory under `data_dir` (a copy where the link fails, - such as across filesystems; cocoon refuses to attach a path inside its own - directories) and attached read-only, and its copy-on-write disk as a private copy that the - reader mounts writable, so the guest replays the ext4 journal onto the copy - and never onto the original; -- the reader mounts the layers and the copy, assembles a read-only overlay at - `/view`, and starts the claim's own `silkd` (`/usr/local/bin/silkd`) chrooted - into it on guest vsock port 2051. Paths and symlinks resolve inside the - claim's filesystem, and a write fails at the read-only overlay; -- a hibernated claim's copy is a reflink export of the snapshot, so keep - `data_dir` and cocoon's data directory on a filesystem that supports - reflinks, as for checkpoints; without it the export copies the guest memory - image too. An archived claim fetches only the record's `cow.raw` and - `cocoon.json` straight into the reader's own directory: the dir backend - copies the file out of the record, the s3 backend downloads those two - objects. Nothing lands in the store's shared fetch cache, so releasing the - reader removes every byte it fetched. - -The reader template must offer `mount`, `chroot`, `env`, `mkdir` and `sync` -and boot a kernel that mounts erofs, ext4 and overlayfs, which the shipped -images do. The claim's image must ship `silkd` at `/usr/local/bin/silkd`, as -the shipped Linux images do; the android image does not, so its claims always -wake. A missing silkd is detected right after the view is mounted, so such a -claim pays the snapshot export, one reader VM and the mounts before its call -wakes it, never a probe timeout. A reader is kept for 30 s after its last connection, then its VM is -destroyed and its copy deleted. A wake waits for a reader only while its -setup exports the hibernate snapshot: milliseconds with a reflink, and the -time to copy the memory image without one. A reader serves only the sleep it -was built from: a later hibernate or archive gets a fresh one. The claim's own code runs in its reader with root, as it did in the -claim; the reader has no NIC, serves one claim, and is destroyed afterwards, so +It needs a cocoon whose `vm disk attach` takes `--from-snapshot` and +`--from-dir` (released after v0.7.2) and Cloud Hypervisor: a claim on a +Firecracker lane always wakes, as does an archive exported by an older cocoon, +whose `snapshot.json` has no `disk_attach_files`. Unpooled claims (template and +checkpoint claims) always wake too. `reader` is reloadable like `capture_trim`. + +A reader is a warm VM taken from the claim's own pool, so it boots the claim's +image with the claim's layers, and it has that pool's lane: no NIC on the none +lane, the lane's NIC otherwise, though sandboxd never arms the claim's egress +policy on it and only the four file calls reach it. Taking one removes a VM from the pool until the refill replaces it, and an empty pool +never cold-boots one: the call wakes the claim. A reader is never shared: it +serves one claim, and only the claim's own token reaches it. The guest's own +kernel does the parsing, as it does for the claim: + +- a hibernated claim's copy-on-write disk is copied by cocoon (`vm disk attach + --from-snapshot`) into the reader VM and hot-attached read-write; cocoon uses + a reflink where the filesystem has them, copies only that disk, never the + memory image, and removes the copy with the VM. The claim's transition lock is + held for the copy so a wake cannot consume the snapshot meanwhile; +- an archived claim's record is fetched only for `snapshot.json` and the files + its `disk_attach_files` lists, into the reader's own directory under + `data_dir` (the dir backend copies them out of the record, the s3 backend + downloads those objects), then attached with `--from-dir`. The fetched files + are deleted once cocoon has its copy, and nothing lands in the store's shared + fetch cache; +- the reader mounts the attached disk writable, so the guest replays the ext4 + journal onto the copy and never onto the original, and mounts read-only the + layers named by its own `cocoon.layers=` kernel parameter. It assembles a + read-only overlay at `/view` and starts the claim's own `silkd` + (`/usr/local/bin/silkd`) chrooted into it on guest vsock port 2051. Paths and + symlinks resolve inside the claim's filesystem, and a write fails at the + read-only overlay. + +The pool's image must offer `mount`, `chroot`, `env`, `mkdir` and `sync` and +boot a kernel that mounts erofs, ext4 and overlayfs, which the shipped images +do. The claim's image must ship `silkd` at `/usr/local/bin/silkd`, as the +shipped Linux images do; the android image does not, so its claims always wake. +A missing silkd is detected right after the view is mounted, so such a claim +pays one reader VM, the copy and the mounts before its call wakes it, never a +probe timeout. A reader is kept for 30 s after its last connection, then its VM +is destroyed. A wake waits for a reader only while cocoon copies the disk: +milliseconds with a reflink, the time to copy the whole disk without one. A +reader serves only the sleep it was built from: a later hibernate or archive +gets a fresh one. The claim's own code runs in its reader with root, as it did +in the claim, and the reader serves one claim and is destroyed afterwards, so the claim's bytes never reach a shared VM or the host kernel. -Whenever a reader cannot be built (no warm reader VM, the disk reserve is low, -a layer blob missing on this node, an image without `silkd` at that path), the -call wakes the claim as it does without the feature. With the feature on, -every hibernate runs a guest `sync` right before the pause, and a reader shows -the disk as of that `sync`: a write the guest makes in the instant between the -`sync` and the pause is visible only after a wake. A hibernate whose `sync` +Whenever a reader cannot be built (no warm VM in the claim's pool, the disk +reserve is low, a cocoon without `--from-snapshot`, a Firecracker lane, an archive without +`disk_attach_files`, an image without `silkd` at that path), the call wakes the +claim as it does without the feature. With the feature on for the pool, every +hibernate of its claims runs a guest `sync` right before the pause, and a reader +shows the disk as of that `sync`: a write the guest makes in the instant between +the `sync` and the pause is visible only after a wake. A hibernate whose `sync` fails or times out is marked not reader-safe, as is any sleep that began before -the feature was on, and its reads wake the claim as without the feature. +the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes @@ -782,7 +787,7 @@ validates it with the same code as boot, and applies the reloadable settings all at once or not at all. Every field falls into one of three classes: - **Reloaded**: - - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `egress_upstream_env`, for existing keys and for keys new in the file; + - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `reader`, `egress_upstream_env`, for existing keys and for keys new in the file; - `egress_classes`: each class's `egress` and `egress_upstream_env`; - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_conn_rate` and `disk_reserve`. - **Left to their API**: a pool's targets (`warm`, `warm_max`, idle and archive diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index f23f1bce..6213582a 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -394,7 +394,7 @@ release). Atomically snapshots the VM and stops it, freeing its memory; the next agent access restores it transparently (sessions, processes, and memory state intact — cocoon's hibernate keeps the snapshot point and the stop coincident), except a read-only file call that a -configured [reader](#get-v1sandboxesidagent) serves without waking it. Idempotent on an already-hibernated +[reader](#get-v1sandboxesidagent) serves without waking it. Idempotent on an already-hibernated sandbox. The TTL keeps running: a hibernated sandbox is still reaped (VM and snapshot) at its deadline, unless it claimed `on_expire: archive` or its pool archives. When to hibernate is the caller's policy — the @@ -579,7 +579,7 @@ is deleted; a pool kept with `warm: 0` keeps its golden. Answers the fresh `GET /v1/info` payload. 400 bad key, negative warm/idle, `warm_max` below `warm`, `idle_hibernate_seconds` on an egress pool, a negative archive duration or an `archive_after_seconds` not above the pool's -`idle_hibernate_seconds`, duplicate pool, or a config-owned `egress`/`warmup`/`capture_trim`/`storage`/`replicate_seconds`/`egress_upstream_env` +`idle_hibernate_seconds`, duplicate pool, or a config-owned `egress`/`warmup`/`capture_trim`/`storage`/`replicate_seconds`/`reader`/`egress_upstream_env` field; 401 bad api token; 409 egress pool on a node without an egress attachment; 503 the `meta_store` cannot take a cell's write. @@ -966,10 +966,10 @@ claim of a replicating pool that is already open when the lease turns overdue is closed, whether or not it has a replica yet. A hibernated or archived sandbox wakes on the first frame that opens an RPC, -unless the node configures a [`reader`](deploy.md#reading-sleeping-claims-without-waking-them). -With one, the node answers the `101` before it touches the sandbox, reads the +unless the sandbox's pool sets [`reader`](deploy.md#reading-sleeping-claims-without-waking-them). +With it, the node answers the `101` before it touches the sandbox, reads the first frame, and serves `fs_stat`, `fs_list`, `fs_read` and `fs_pull` from a -disposable reader VM that mounts the sandbox's disk read-only: the sandbox stays +reader VM, a warm VM of the sandbox's pool that mounts a copy of the sandbox's disk read-only: the sandbox stays hibernated or archived, no `wake` usage event is recorded, and the read stamps neither the idle nor the archive clock. An `info` frame pipelined ahead of the call (the SDKs' first frames) rides along to the reader; one alone, with no call @@ -983,8 +983,8 @@ from a stale sleep. Files and symlinks resolve inside the sandbox's own filesystem as of the guest `sync` that runs right before the last hibernate's pause; a write the guest made in the instant between that `sync` and the pause is visible only after a wake. A -hibernate whose `sync` failed, or one made before the reader was configured, -is not served by a reader: its reads wake the sandbox. When no reader can be built the call wakes the sandbox as if the +hibernate whose `sync` failed, or one made before the pool had `reader`, +is not served by a reader: its reads wake the sandbox. When no reader can be built (no warm VM in the pool, a Firecracker lane, an archive from an older cocoon) the call wakes the sandbox as if the feature were off. Because the `101` has gone out by then, a failure on this path (unknown sandbox, a refused wake, an unreachable guest) arrives as a silkd `error` frame (`not_found` for an unknown sandbox, else `internal`) rather than diff --git a/docs/sdk.md b/docs/sdk.md index 2970bcb0..2d66f2aa 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -376,7 +376,7 @@ automatically with the same transparent wake. A claim with a connection live when the sweep checks it (a relay stream, a buffered exec, a preview dial, an egress request) is not swept; the idle clock restarts when that connection ends. -When the node configures a [`reader`](deploy.md#reading-sleeping-claims-without-waking-them), +When the sandbox's pool sets [`reader`](deploy.md#reading-sleeping-claims-without-waking-them), `Stat`, `ReadFile`, `ReadFileTo`, `ListDir` and `Pull` on a hibernated or archived handle are served without waking it, as of the guest `sync` just before the last hibernate, and the handle's next call that is anything else wakes the sandbox on the same connection. From e9c854fdf2b1c371fda9c66274ef1cca606fd046 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 12:56:37 +0800 Subject: [PATCH 21/28] docs: reader is off on Firecracker and older cocoon; a bridge-lane reader keeps the claim's NIC --- docs/deploy.md | 20 +++++++++++--------- docs/sandboxd-api.md | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 5d7d66d5..a1130bdc 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -185,17 +185,19 @@ the claim as before ([API](sandboxd-api.md#get-v1sandboxesidagent)). } ``` -It needs a cocoon whose `vm disk attach` takes `--from-snapshot` and -`--from-dir` (released after v0.7.2) and Cloud Hypervisor: a claim on a -Firecracker lane always wakes, as does an archive exported by an older cocoon, -whose `snapshot.json` has no `disk_attach_files`. Unpooled claims (template and -checkpoint claims) always wake too. `reader` is reloadable like `capture_trim`. +It needs Cloud Hypervisor and a cocoon whose `vm disk attach` takes +`--from-snapshot` and `--from-dir` (released after v0.7.2). Leave `reader` off +on a Firecracker node or with an older cocoon: there every read spends a warm VM +on an attach that fails, then wakes the claim. An archive exported by an older +cocoon, whose `snapshot.json` has no `disk_attach_files`, wakes before any VM is +taken. Unpooled claims (template and checkpoint claims) always wake. +`reader` is reloadable like `capture_trim`. A reader is a warm VM taken from the claim's own pool, so it boots the claim's image with the claim's layers, and it has that pool's lane: no NIC on the none -lane, the lane's NIC otherwise, though sandboxd never arms the claim's egress -policy on it and only the four file calls reach it. Taking one removes a VM from the pool until the refill replaces it, and an empty pool -never cold-boots one: the call wakes the claim. A reader is never shared: it +lane, the bridge NIC the claim had on the bridge lane (egress-lane claims never +hibernate). Taking one removes a VM from the pool until the refill replaces it, +and an empty pool never cold-boots one: the call wakes the claim. A reader is never shared: it serves one claim, and only the claim's own token reaches it. The guest's own kernel does the parsing, as it does for the claim: @@ -233,7 +235,7 @@ in the claim, and the reader serves one claim and is destroyed afterwards, so the claim's bytes never reach a shared VM or the host kernel. Whenever a reader cannot be built (no warm VM in the claim's pool, the disk -reserve is low, a cocoon without `--from-snapshot`, a Firecracker lane, an archive without +reserve is low, a failed attach, an archive without `disk_attach_files`, an image without `silkd` at that path), the call wakes the claim as it does without the feature. With the feature on for the pool, every hibernate of its claims runs a guest `sync` right before the pause, and a reader diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 6213582a..ec11d3ff 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -984,7 +984,7 @@ and symlinks resolve inside the sandbox's own filesystem as of the guest `sync` that runs right before the last hibernate's pause; a write the guest made in the instant between that `sync` and the pause is visible only after a wake. A hibernate whose `sync` failed, or one made before the pool had `reader`, -is not served by a reader: its reads wake the sandbox. When no reader can be built (no warm VM in the pool, a Firecracker lane, an archive from an older cocoon) the call wakes the sandbox as if the +is not served by a reader: its reads wake the sandbox. When no reader can be built (no warm VM in the pool, a failed attach, an archive from an older cocoon) the call wakes the sandbox as if the feature were off. Because the `101` has gone out by then, a failure on this path (unknown sandbox, a refused wake, an unreachable guest) arrives as a silkd `error` frame (`not_found` for an unknown sandbox, else `internal`) rather than From 6d1bfab79f9f39607473da70571ad89bf9ab1791 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 13:01:27 +0800 Subject: [PATCH 22/28] docs: turning reader off leaves open reader connections to finish --- docs/deploy.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index a1130bdc..c040d11f 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -191,7 +191,8 @@ on a Firecracker node or with an older cocoon: there every read spends a warm VM on an attach that fails, then wakes the claim. An archive exported by an older cocoon, whose `snapshot.json` has no `disk_attach_files`, wakes before any VM is taken. Unpooled claims (template and checkpoint claims) always wake. -`reader` is reloadable like `capture_trim`. +`reader` is reloadable like `capture_trim`; turning it off leaves open reader +connections to finish, and the next connection wakes the claim. A reader is a warm VM taken from the claim's own pool, so it boots the claim's image with the claim's layers, and it has that pool's lane: no NIC on the none From 9ee236d218b4cde6d07ad89042b22702939ff1f6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:18:51 +0800 Subject: [PATCH 23/28] reader: serve and relay sleeping claims under the node lease fence A replicated claim's reader dial skips the fence the wake path applies, and a reader connection or a handed-over relay of a sleeping claim stays open after the lease turns overdue. DialReader now gates through fenceWatch and closes its connection on a trip, and the router's openGuest closes the guest the same way wakeGuest does. --- sandboxd/pool/pool_test.go | 9 +++-- sandboxd/pool/reader.go | 7 +++- sandboxd/pool/reader_test.go | 65 ++++++++++++++++++++++++++++++++++ sandboxd/server/reader.go | 6 ++-- sandboxd/server/reader_test.go | 16 +++++++++ sandboxd/server/relay.go | 14 +++++--- 6 files changed, 106 insertions(+), 11 deletions(-) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index c24d783e..10c7c81e 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1229,6 +1229,7 @@ type fakeEngine struct { readerDials []string startReaderErr error readerDialErr error + readerPeers chan net.Conn syncErr error caInstalls []string warmups [][]string @@ -1731,13 +1732,17 @@ func (f *fakeEngine) StartReader(_ context.Context, vsockSocket, cowDisk string) func (f *fakeEngine) DialReader(_ context.Context, vsockSocket string) (net.Conn, error) { f.mu.Lock() f.readerDials = append(f.readerDials, vsockSocket) - err := f.readerDialErr + err, peers := f.readerDialErr, f.readerPeers f.mu.Unlock() if err != nil { return nil, err } conn, peer := net.Pipe() - _ = peer.Close() + if peers == nil { + _ = peer.Close() + } else { + peers <- peer + } return conn, nil } diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index cfeb6970..8e598028 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -32,6 +32,10 @@ func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Co if !ok { return nil, "", ErrUnknownSandbox } + fenced, err := m.fenceWatch(sb) + if err != nil { + return nil, "", err + } if !m.view.Load().poolReaders[sb.Key] { return nil, "", ErrNoReader } @@ -45,7 +49,8 @@ func (m *Manager) DialReader(ctx context.Context, id, token string) (conn net.Co m.dropReader(ctx, r, true) return nil, "", fmt.Errorf("%w: %w", ErrNoReader, err) } - return &heldConn{Conn: conn, release: sync.OnceFunc(func() { m.releaseReader(r) })}, r.source, nil + stop := CloseOnFence(fenced, conn) + return &heldConn{Conn: conn, release: sync.OnceFunc(func() { stop(); m.releaseReader(r) })}, r.source, nil } // SleepingSource names the hibernate snapshot or archive record a reader would serve, or "" when the claim is awake, failed or mid-transition. diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index e7961033..9a4e1067 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "os" "path/filepath" "slices" @@ -57,6 +58,70 @@ func TestReaderServesAHibernatedClaimWithoutWaking(t *testing.T) { } } +func TestReaderObeysTheNodeLeaseFence(t *testing.T) { + eng := newFakeEngine() + m := newReaderManager(t, eng) + replicated, plain := mustClaim(t, m, testKey), mustClaim(t, m, testKey) + replicated.SetReplicated(true) + hibernateClaim(t, m, replicated) + hibernateClaim(t, m, plain) + seedReaderVMs(m, testKey, "sbx-reader-1", "sbx-reader-2") + m.tripFence() + + if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { + t.Errorf("WakeAgentSocketOrRead of a replicated sleeper past the fence: %v, want ErrLeaseLost", err) + } + if _, _, err := m.DialReader(t.Context(), replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { + t.Fatalf("DialReader of a replicated sleeper past the fence: %v, want ErrLeaseLost", err) + } + if len(eng.attachedCopies) != 0 { + t.Errorf("a fenced read set up a reader: %v", eng.attachedCopies) + } + conn, _, err := m.DialReader(t.Context(), plain.ID, plain.Token) + if err != nil { + t.Fatalf("DialReader of an unreplicated sleeper past the fence: %v", err) + } + _ = conn.Close() +} + +func TestReaderConnectionEndsWhenTheFenceTrips(t *testing.T) { + eng := newFakeEngine() + eng.readerPeers = make(chan net.Conn, 2) + m := newReaderManager(t, eng) + m.liftFence() + replicated, plain := mustClaim(t, m, testKey), mustClaim(t, m, testKey) + replicated.SetReplicated(true) + hibernateClaim(t, m, replicated) + hibernateClaim(t, m, plain) + seedReaderVMs(m, testKey, "sbx-reader-1", "sbx-reader-2") + + peers := map[string]net.Conn{} + for name, sb := range map[string]*types.Sandbox{"replicated": replicated, "plain": plain} { + conn, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader of the %s claim: %v", name, err) + } + defer func() { _ = conn.Close() }() + peers[name] = <-eng.readerPeers + } + ended := func(peer net.Conn, wait time.Duration) bool { + _ = peer.SetReadDeadline(time.Now().Add(wait)) + _, err := peer.Read(make([]byte, 1)) + return !errors.Is(err, os.ErrDeadlineExceeded) + } + if ended(peers["replicated"], 100*time.Millisecond) || ended(peers["plain"], 100*time.Millisecond) { + t.Fatal("a reader connection ended before the fence tripped") + } + + m.trip(t.Context()) + if !ended(peers["replicated"], 3*time.Second) { + t.Error("the replicated claim's reader connection survived the fence") + } + if ended(peers["plain"], 100*time.Millisecond) { + t.Error("the fence ended an unreplicated claim's reader connection") + } +} + func TestReaderTakesItsVMFromTheClaimsOwnPool(t *testing.T) { eng := newFakeEngine() m := newReaderManager(t, eng, config.PoolSpec{PoolKey: otherKey, Warm: 1}) diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index 411faf63..2862ff19 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -88,7 +88,7 @@ func (r *router) run() (net.Conn, func(), bool) { case err == nil: case errors.Is(err, errReaderGone): return nil, nil, false - case errors.Is(err, pool.ErrUnknownSandbox): + case errors.Is(err, pool.ErrUnknownSandbox), errors.Is(err, pool.ErrLeaseLost): r.fail(err) return nil, nil, false default: @@ -274,7 +274,7 @@ func (s *Server) serveSleeping(ctx context.Context, w http.ResponseWriter, id, t } func (s *Server) openGuest(ctx context.Context, id, token string) (net.Conn, func(), error) { - sock, _, done, err := s.mgr.WakeAgentSocket(ctx, id, token) + sock, fenced, done, err := s.mgr.WakeAgentSocket(ctx, id, token) if err != nil { return nil, nil, err } @@ -283,5 +283,5 @@ func (s *Server) openGuest(ctx context.Context, id, token string) (net.Conn, fun done() return nil, nil, fmt.Errorf("%w: %w", errUnreachable, err) } - return guest, done, nil + return guest, fenceGuest(guest, fenced, done), nil } diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go index 52248f9d..71f2c75d 100644 --- a/sandboxd/server/reader_test.go +++ b/sandboxd/server/reader_test.go @@ -188,6 +188,7 @@ func TestSleepingFailuresAnswerWithAnErrorFrame(t *testing.T) { message string }{ {"unknown sandbox", pool.ErrUnknownSandbox, nil, statFrame, wire.KindNotFound, "unknown sandbox"}, + {"lease lost", pool.ErrLeaseLost, nil, statFrame, wire.KindInternal, "lease lost; retry"}, {"wake refused", pool.ErrNoReader, pool.ErrTenantRemoved, statFrame, wire.KindInternal, pool.ErrTenantRemoved.Error()}, {"wake fails", nil, errors.New("restore broke"), execFrame, wire.KindInternal, "sandbox lookup failed"}, } { @@ -216,6 +217,21 @@ func TestSleepingFailuresAnswerWithAnErrorFrame(t *testing.T) { } } +func TestSleepingHandoverEndsWhenItsClaimIsFenced(t *testing.T) { + env := newSleepingEnv(t) + fenced := make(chan struct{}) + env.manager.fenced = fenced + conn, r := env.upgrade(t) + send(t, conn, execFrame) + expectLine(t, r, "guest") + + close(fenced) + _ = conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, err := r.ReadByte(); !errors.Is(err, io.EOF) { + t.Errorf("handed-over relay after the fence: %v, want EOF", err) + } +} + func TestSleepingFramesAreAuditedOnceOnEitherPath(t *testing.T) { env := newSleepingEnv(t) var mu sync.Mutex diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 9cf4c7a6..1440a23f 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -100,11 +100,7 @@ func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token writeErr(w, http.StatusBadGateway, "guest agent unreachable") return nil, nil, false, false } - if fenced == nil { - return guest, done, false, true - } - stop := pool.CloseOnFence(fenced, guest) - return guest, func() { stop(); done() }, false, true + return guest, fenceGuest(guest, fenced, done), false, true } func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBuf *bufio.Reader, guest net.Conn) { @@ -211,6 +207,14 @@ func (t *auditTee) endLine() { t.skip = false } +func fenceGuest(guest net.Conn, fenced <-chan struct{}, done func()) func() { + if fenced == nil { + return done + } + stop := pool.CloseOnFence(fenced, guest) + return func() { stop(); done() } +} + func copyBoth(client net.Conn, clientR io.Reader, guest net.Conn, ends relayEnds) { done := make(chan struct{}) go func() { From 43213f7abce1adb30bca0c6c9f0eeb0662571ea4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:20:01 +0800 Subject: [PATCH 24/28] pool: a hibernate that left no snapshot clears reader_safe The flag is set before the engine call, so a failed hibernate left it on a running claim, and a replica row written from that claim carried it into an adoption that would then serve the replica from a reader. --- sandboxd/pool/hibernate.go | 2 ++ sandboxd/pool/reader_test.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index cac827ed..aeebb90e 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -356,6 +356,8 @@ func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (ad if live { if exists { sb.HibernateSnap = pending + } else { + sb.ReaderSafe = false } sb.PendingSnap = "" js = m.store.set(sb) diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 9a4e1067..5aaab43c 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -699,6 +699,23 @@ func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { } } +func TestFailedHibernateLeavesNoReaderSafeOnARunningClaim(t *testing.T) { + eng := newFakeEngine() + eng.hibernateErr = errors.New("snapshot failed") + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("hibernate succeeded, want the engine's failure") + } + if sb.HibernateSnap != "" || sb.PendingSnap != "" { + t.Fatalf("claim state after the failed hibernate: snap %q pending %q, want a running claim", sb.HibernateSnap, sb.PendingSnap) + } + if sb.ReaderSafe { + t.Error("a running claim keeps reader_safe from the hibernate that failed, and a replica row would carry it") + } +} + func TestWakeClearsReaderSafe(t *testing.T) { eng := newFakeEngine() m := newReaderManager(t, eng) From fe70617dc98888ed32fd7016ea98bf9e869bf656 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:20:35 +0800 Subject: [PATCH 25/28] docs: reader needs cocoon v0.7.3, and an overdue node lease ends reader reads of replicated claims --- docs/cluster.md | 4 +++- docs/deploy.md | 4 ++-- docs/sandboxd-api.md | 8 +++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 1f1bafb9..2989113a 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -334,7 +334,9 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease length minus a 3 second margin, counted from the start of the last renewal that answered) stops serving every replicated claim it holds: relay, exec, - ports and preview answer `503 lease lost; retry`, and `/owner` answers 404 so + ports and preview answer `503 lease lost; retry` (a read through a + [reader](deploy.md#reading-sleeping-claims-without-waking-them) after the `101` + gets an `error` frame instead), and `/owner` answers 404 so a `Lookup` moves on to the other nodes. A claim is replicated when its policy pool has `replicate_seconds`, from the moment it is claimed, adopted or loaded, whether or not a round has run, so the fence covers it before its diff --git a/docs/deploy.md b/docs/deploy.md index c040d11f..caed0f4a 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -14,7 +14,7 @@ the cocoon CLI and needs a template image with silkd baked in. empty `snapshot list --format json`, which sandboxd's snapshot reconcile reads. sandboxd logs a warning at startup when the detected cocoon is below v0.6.9 (a dev/`master-` build is assumed current); changing volumes on - a running claim needs v0.7.2 + a running claim needs v0.7.2, and a pool's `reader` needs v0.7.3 - The sandbox boot artifact installed where cocoon finds it (`/boot/vmlinuz-sandbox`, `/boot/initrd.img-sandbox` — from `ghcr.io/cocoonstack/sandbox/boot:`) @@ -186,7 +186,7 @@ the claim as before ([API](sandboxd-api.md#get-v1sandboxesidagent)). ``` It needs Cloud Hypervisor and a cocoon whose `vm disk attach` takes -`--from-snapshot` and `--from-dir` (released after v0.7.2). Leave `reader` off +`--from-snapshot` and `--from-dir` (v0.7.3 or newer). Leave `reader` off on a Firecracker node or with an older cocoon: there every read spends a warm VM on an attach that fails, then wakes the claim. An archive exported by an older cocoon, whose `snapshot.json` has no `disk_attach_files`, wakes before any VM is diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ec11d3ff..31a255cd 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -986,9 +986,11 @@ the instant between that `sync` and the pause is visible only after a wake. A hibernate whose `sync` failed, or one made before the pool had `reader`, is not served by a reader: its reads wake the sandbox. When no reader can be built (no warm VM in the pool, a failed attach, an archive from an older cocoon) the call wakes the sandbox as if the feature were off. Because the `101` has gone out by then, a failure on this path -(unknown sandbox, a refused wake, an unreachable guest) arrives as a silkd -`error` frame (`not_found` for an unknown sandbox, else `internal`) rather than -an HTTP status, and the connection ends. +(unknown sandbox, a refused wake, an unreachable guest, an overdue node lease) +arrives as a silkd `error` frame (`not_found` for an unknown sandbox, else +`internal`) rather than an HTTP status, and the connection ends. A connection +served by a reader, or handed over to the woken sandbox, of a claim of a +replicating pool is closed when the lease turns overdue, as an open relay is. ## GET /v1/sandboxes/{id}/ports/{port} From 06defdee6f98265d9da040a55cb5c595a74cdd31 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:37:19 +0800 Subject: [PATCH 26/28] docs: a reader's archive fetch downloads only the chunks of the files it needs --- docs/deploy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index caed0f4a..f8020f0e 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -210,7 +210,7 @@ kernel does the parsing, as it does for the claim: - an archived claim's record is fetched only for `snapshot.json` and the files its `disk_attach_files` lists, into the reader's own directory under `data_dir` (the dir backend copies them out of the record, the s3 backend - downloads those objects), then attached with `--from-dir`. The fetched files + downloads only the chunks those files name, holes stay holes), then attached with `--from-dir`. The fetched files are deleted once cocoon has its copy, and nothing lands in the store's shared fetch cache; - the reader mounts the attached disk writable, so the guest replays the ext4 From 0fa4c7df9cf937d4d4a5dc36a41118c64f0d2b84 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:58:38 +0800 Subject: [PATCH 27/28] pool: a replica row never carries reader_safe A replica checkpoint captures a running claim and is never synced for a reader, but its row copied the claim's flag, which a replication round racing a hibernate or a restart that kept the flag over a restored wake leaves true. Adoption would then serve the unsynced disk from a reader. putRow clears the flag on the row's copy for replica rows; archive rows keep it. This replaces clearing the flag after a failed hibernate, whose only harm was reaching a replica row. --- sandboxd/pool/failover.go | 1 + sandboxd/pool/hibernate.go | 2 -- sandboxd/pool/reader_test.go | 17 ---------- sandboxd/pool/replica_test.go | 60 +++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 4bb0b450..dd0896f4 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -140,6 +140,7 @@ func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, repl m.mu.Unlock() dto.Token, dto.VMName, dto.VsockSocket, dto.HibernateSnap, dto.PendingSnap, dto.ArchivePending, dto.ReplicaCk, dto.ReplicaPending = "", "", "", "", "", "", "", "" dto.ArchiveCk, dto.Deadline = ck, deadline + dto.ReaderSafe = dto.ReaderSafe && !replica claim, err := json.Marshal(&dto) if err != nil { return 0, fmt.Errorf("encode claim: %w", err) diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index aeebb90e..cac827ed 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -356,8 +356,6 @@ func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (ad if live { if exists { sb.HibernateSnap = pending - } else { - sb.ReaderSafe = false } sb.PendingSnap = "" js = m.store.set(sb) diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 5aaab43c..9a4e1067 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -699,23 +699,6 @@ func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { } } -func TestFailedHibernateLeavesNoReaderSafeOnARunningClaim(t *testing.T) { - eng := newFakeEngine() - eng.hibernateErr = errors.New("snapshot failed") - m := newReaderManager(t, eng) - sb := mustClaim(t, m, testKey) - - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { - t.Fatal("hibernate succeeded, want the engine's failure") - } - if sb.HibernateSnap != "" || sb.PendingSnap != "" { - t.Fatalf("claim state after the failed hibernate: snap %q pending %q, want a running claim", sb.HibernateSnap, sb.PendingSnap) - } - if sb.ReaderSafe { - t.Error("a running claim keeps reader_safe from the hibernate that failed, and a replica row would carry it") - } -} - func TestWakeClearsReaderSafe(t *testing.T) { eng := newFakeEngine() m := newReaderManager(t, eng) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 0de3be9e..e36dee45 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -216,6 +216,66 @@ func TestReplicaRowOfAHibernatedClaimHasNoRetentionDeadline(t *testing.T) { }) } +func TestReplicaRowIsNeverReaderSafe(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + a.mu.Lock() + sb.ReaderSafe = true + a.mu.Unlock() + + replicateNow(t, a, sb) + row, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok { + t.Fatalf("row: %v %v, want one", ok, err) + } + claim := &types.Sandbox{} + if err = json.Unmarshal(row.Claim, claim); err != nil || claim.ReaderSafe { + t.Errorf("replica row claim reader_safe %v (%v), want false", claim.ReaderSafe, err) + } + if !sb.ReaderSafe { + t.Error("writing the row cleared the live claim's reader_safe") + } + + c.expire("a") + if adopted, adoptErr := b.AdoptArchived(t.Context(), sb.ID, sb.Token); adoptErr != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, adoptErr) + } + sock, _, done, err := b.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token) + if err != nil || sock == "" { + t.Fatalf("read of the adopted replica = %q, %v, want a wake", sock, err) + } + done() +} + +func TestArchiveRowKeepsReaderSafe(t *testing.T) { + c := newFailoverCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + if err := a.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } + a.mu.Lock() + sb.ReaderSafe = true + a.mu.Unlock() + if err := a.archive(t.Context(), sb); err != nil { + t.Fatalf("archive: %v", err) + } + + row, _, _ := c.reg.Get(t.Context(), sb.ID) + claim := &types.Sandbox{} + if err := json.Unmarshal(row.Claim, claim); err != nil || !claim.ReaderSafe { + t.Errorf("archive row claim reader_safe %v (%v), want true", claim.ReaderSafe, err) + } + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, err) + } + if _, _, _, err := b.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + t.Errorf("read of the adopted archive = %v, want ErrSleeping", err) + } +} + func TestReplicaRetireAfterAnArchiveLeavesTheArchiveRowAlone(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) From 6a5daca0dff2c90850c1214a2d250a7259227799 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 15:50:16 +0800 Subject: [PATCH 28/28] server: a fresh reader's pump closes the client again closeReader marks the old pump superseded so it leaves the client open; the flag stayed set for the next reader, so a fresh reader that a fence or a crash ended never closed the client and an in-flight read hung until the client gave up. --- sandboxd/server/reader.go | 1 + sandboxd/server/reader_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index 2862ff19..fa0cc635 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -189,6 +189,7 @@ func (r *router) infoFollowed() (bool, error) { } func (r *router) startPump() { + r.superseded.Store(false) r.pumped = make(chan struct{}) reader := r.reader go func() { diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go index 71f2c75d..08ed090f 100644 --- a/sandboxd/server/reader_test.go +++ b/sandboxd/server/reader_test.go @@ -156,6 +156,26 @@ func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing } } +func TestSleepingFreshReaderEndClosesTheClient(t *testing.T) { + env := newSleepingEnv(t) + conn, r := env.upgrade(t) + send(t, conn, statFrame) + expectLine(t, r, "reader") + + env.reader.mu.Lock() + env.reader.src = "snap-2" + env.reader.serve = func(c net.Conn) { + _, _ = bufio.NewReader(c).ReadString('\n') + _ = c.Close() + } + env.reader.mu.Unlock() + send(t, conn, readFrame) + _ = conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, err := r.ReadString('\n'); !errors.Is(err, io.EOF) { + t.Errorf("client after its fresh reader ended: %v, want EOF", err) + } +} + func TestSleepingUnavailableReaderWakes(t *testing.T) { logs := captureLogs(t) env := newSleepingEnv(t)