From 109599d11978fba5173db6218a8384a1c60efd0e Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Sat, 17 Oct 2026 03:48:21 +0800 Subject: [PATCH 1/2] fix(vmm): offload recursive directory deletion Signed-off-by: Leechael Yim --- dstack/vmm/src/app.rs | 301 +++++++++++++++++++++++++++++++-- dstack/vmm/src/main_service.rs | 97 ++++++++++- 2 files changed, 375 insertions(+), 23 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 5baf6aeae..1d2f06905 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -353,6 +353,8 @@ pub struct App { /// One lock per VM, held across a launch or a teardown. See /// [`App::launch_lock`]. launch_locks: Arc>>>>, + #[cfg(test)] + before_remove_workdir: Option Result<()> + Send + Sync>>, } const GUEST_AGENT_RPC_TIMEOUT: Duration = Duration::from_secs(30); @@ -384,6 +386,8 @@ impl App { })), config: Arc::new(config), launch_locks: Arc::new(Mutex::new(HashMap::new())), + #[cfg(test)] + before_remove_workdir: None, } } @@ -1050,6 +1054,14 @@ impl App { Ok(()) } + fn remove_vm_workdir(&self, workdir: &VmWorkDir) -> Result<()> { + #[cfg(test)] + if let Some(before_remove) = &self.before_remove_workdir { + before_remove()?; + } + workdir.remove_all() + } + /// Background cleanup: stop supervisor process, wait for it to exit, /// remove from supervisor, optionally delete workdir, and free CID. /// @@ -1067,7 +1079,7 @@ impl App { // launch went on to start. Taking the lock first means the launch // finishes before removal decides anything, and removal then stops what // it actually started. - let _launch = self.launch_lock(id).await; + let launch = self.launch_lock(id).await; // Stop the supervisor process (idempotent if already stopped) if let Err(err) = self.stop_vm_process(id).await { debug!("graceful VM stop during removal failed: {err:?}"); @@ -1112,7 +1124,8 @@ impl App { let vm_path = self.work_dir(id)?; // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. - if !(delete_workdir || vm_path.is_removing()) { + let delete_workdir = delete_workdir || vm_path.is_removing(); + if !delete_workdir { self.release_vm_interfaces(id).await; if vm_path.path().exists() { info!( @@ -1134,22 +1147,35 @@ impl App { tokio::time::sleep(delay).await; delay = (delay * 2).min(RELEASE_RETRY_MAX); } - if let Err(err) = vm_path.remove_all() { - // `.removing` is deleted last, so what is left is resumed by - // the next reload. - error!("failed to remove VM directory for {id}: {err:?}"); - } } - // Free CID and remove from memory (last step) - { - let mut state = self.lock(); - if let Some(vm_state) = state.remove(id) { - state.cid_pool.free(vm_state.config.cid); + // Unlinking a preallocated disk can spend seconds freeing extents. Keep + // it off the async executor. The guards and final state cleanup travel + // with the deletion: cancelling the waiter must neither let another + // launch race it nor leave a deleted VM occupying its CID in memory. + let app = self.clone(); + let id = id.to_string(); + tokio::task::spawn_blocking(move || { + let _launch = launch; + let _mark = mark; + if delete_workdir { + if let Err(err) = app.remove_vm_workdir(&vm_path) { + // `.removing` is deleted last, so what is left is resumed + // by the next reload. + error!("failed to remove VM directory for {id}: {err:?}"); + } } - } - - info!("VM {id} removed successfully"); + // Free CID and remove from memory (last step). + { + let mut state = app.lock(); + if let Some(vm_state) = state.remove(&id) { + state.cid_pool.free(vm_state.config.cid); + } + } + info!("VM {id} removed successfully"); + }) + .await + .context("VM removal task failed")?; Ok(()) } @@ -2377,6 +2403,8 @@ mod tests { let (mut stream, _) = listener.accept().await.unwrap(); let mut reader = BufReader::new(&mut stream); let mut header = String::new(); + assert!(reader.read_line(&mut header).await.unwrap() > 0); + let is_list = header.starts_with("GET /list "); loop { header.clear(); assert!(reader.read_line(&mut header).await.unwrap() > 0); @@ -2384,9 +2412,16 @@ mod tests { break; } } - let body = serde_json::to_string(&supervisor_client::supervisor::Response::Data( - Option::::None, - )) + let body = if is_list { + serde_json::to_string(&supervisor_client::supervisor::Response::Data(Vec::< + supervisor_client::supervisor::ProcessInfo, + >::new( + ))) + } else { + serde_json::to_string(&supervisor_client::supervisor::Response::Data( + Option::::None, + )) + } .unwrap(); stream .write_all( @@ -2412,6 +2447,236 @@ mod tests { workdir } + /// Simulates a slow filesystem operation without allocating production-sized + /// disks. The OS-thread timeout releases even a blocked single-worker runtime. + struct DeletionGate { + entered: tokio::sync::oneshot::Receiver<()>, + release: std::sync::mpsc::Sender<()>, + expired: Arc, + } + + impl DeletionGate { + fn new(app: &mut App) -> Self { + let (entered, entered_rx) = tokio::sync::oneshot::channel(); + let entered = Mutex::new(Some(entered)); + let (release, release_rx) = std::sync::mpsc::channel(); + let release_rx = Mutex::new(release_rx); + let expired = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let timed_out = expired.clone(); + app.before_remove_workdir = Some(Arc::new(move || { + entered.lock().unwrap().take().unwrap().send(()).unwrap(); + if release_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .is_err() + { + timed_out.store(true, std::sync::atomic::Ordering::SeqCst); + } + Ok(()) + })); + Self { + entered: entered_rx, + release, + expired, + } + } + } + + /// Regression: slow workdir deletion must not stop unrelated async work. + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn slow_vm_deletion_does_not_block_status() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let netd = netd::testing::FakeNetd::spawn_at(&dir.path().join("netd.sock"), &["RemoveVm"]); + let mut app = App::new(test_config(netd.socket(), dir.path()), supervisor); + let gate = DeletionGate::new(&mut app); + let workdir = removable_vm(&app, "vm-1"); + workdir.mark_network_cleanup_pending().unwrap(); + + app.remove_vm("vm-1").await.unwrap(); + gate.entered.await.unwrap(); + // The only async worker must be able to run another task while the + // filesystem operation is still paused on a different OS thread. + tokio::spawn(async { tokio::task::yield_now().await }) + .await + .unwrap(); + let status = app + .list_vms(StatusRequest { + ids: vec!["unrelated-vm".into()], + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(status.total, 0); + assert!(status.vms.is_empty()); + assert!( + !gate.expired.load(std::sync::atomic::Ordering::SeqCst), + "Status only progressed after the deletion watchdog released the worker" + ); + assert!(workdir.is_removing()); + assert!(workdir.hda_path().exists()); + assert!(!workdir.network_cleanup_pending()); + assert!(app.refuse_if_removing("vm-1").is_err()); + assert!(app.launch_lock_handle("vm-1").try_lock().is_err()); + assert!(app.launch_lock_handle("unrelated-vm").try_lock().is_ok()); + app.remove_vm("vm-1").await.unwrap(); + assert!(app.start_vm("vm-1").await.is_err()); + assert!(app.lock().get("vm-1").is_some()); + assert!(app + .lock() + .cid_pool + .occupy(app.config.cvm.cid_start) + .is_err()); + assert_eq!(netd.methods(), vec!["RemoveVm"]); + + gate.release.send(()).unwrap(); + let _finished = tokio::time::timeout(Duration::from_secs(5), app.launch_lock("vm-1")) + .await + .unwrap(); + assert!(!workdir.path().exists()); + assert!(app.refuse_if_removing("vm-1").is_ok()); + let mut state = app.lock(); + assert!(state.get("vm-1").is_none()); + assert_eq!(state.cid_pool.allocate(), Some(app.config.cvm.cid_start)); + server.abort(); + } + + fn removable_vm(app: &App, id: &str) -> VmWorkDir { + let workdir = app.work_dir(id).unwrap(); + std::fs::create_dir_all(workdir.shared_dir()).unwrap(); + std::fs::write(workdir.hda_path(), b"test disk contents").unwrap(); + std::fs::write(workdir.shared_dir().join("config"), b"test config").unwrap(); + let (mut vm, _) = bridge_vm(app, id); + { + let mut state = app.lock(); + vm.cid = state.cid_pool.allocate().unwrap(); + state.add(VmState::new(vm)); + } + workdir + } + + /// Invariant: cancelling the waiter must not release exclusion or skip the + /// final state/CID cleanup while an uncancellable filesystem task continues. + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn cancelling_vm_deletion_keeps_guards_and_finishes_cleanup() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let mut app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + let gate = DeletionGate::new(&mut app); + let workdir = removable_vm(&app, "vm-1"); + workdir.set_removing().unwrap(); + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + let removal = tokio::spawn({ + let app = app.clone(); + async move { app.finish_remove_vm(mark, true).await } + }); + gate.entered.await.unwrap(); + removal.abort(); + assert!(removal.await.unwrap_err().is_cancelled()); + assert!(!gate.expired.load(std::sync::atomic::Ordering::SeqCst)); + assert!(workdir.is_removing()); + assert!(workdir.hda_path().exists()); + assert!(app.refuse_if_removing("vm-1").is_err()); + assert!(app.launch_lock_handle("vm-1").try_lock().is_err()); + app.remove_vm("vm-1").await.unwrap(); + assert!(app.lock().get("vm-1").is_some()); + assert!(app + .lock() + .cid_pool + .occupy(app.config.cvm.cid_start) + .is_err()); + + gate.release.send(()).unwrap(); + let _finished = tokio::time::timeout(Duration::from_secs(5), app.launch_lock("vm-1")) + .await + .unwrap(); + assert!(!workdir.path().exists()); + assert!(app.refuse_if_removing("vm-1").is_ok()); + let mut state = app.lock(); + assert!(state.get("vm-1").is_none()); + assert_eq!(state.cid_pool.allocate(), Some(app.config.cvm.cid_start)); + server.abort(); + } + + /// Invariant: an ordinary deletion failure retains `.removing` for reload, + /// while preserving the existing behavior of removing the VM from memory. + #[tokio::test] + async fn failed_vm_deletion_leaves_a_recoverable_marker() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let mut app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + let workdir = removable_vm(&app, "vm-1"); + workdir.set_removing().unwrap(); + app.before_remove_workdir = Some(Arc::new(|| anyhow::bail!("simulated filesystem error"))); + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + app.finish_remove_vm(mark, true).await.unwrap(); + assert!(workdir.is_removing()); + assert!(workdir.hda_path().exists()); + assert!(app.refuse_if_removing("vm-1").is_ok()); + assert!(app.lock().get("vm-1").is_none()); + assert_eq!( + app.lock().cid_pool.allocate(), + Some(app.config.cvm.cid_start) + ); + + app.before_remove_workdir = None; + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + // Reload/orphan cleanup uses the persisted marker, not delete_workdir. + app.finish_remove_vm(mark, false).await.unwrap(); + assert!(!workdir.path().exists()); + server.abort(); + } + + /// Invariant: a blocking-task panic is distinguishable from an ordinary + /// filesystem failure, releases guards, and keeps the deletion retryable. + #[tokio::test] + async fn panicking_vm_deletion_releases_guards_and_reports_task_failure() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let mut app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + let workdir = removable_vm(&app, "vm-1"); + workdir.set_removing().unwrap(); + app.before_remove_workdir = Some(Arc::new(|| panic!("simulated deletion panic"))); + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + let error = app.finish_remove_vm(mark, true).await.unwrap_err(); + assert_eq!(error.to_string(), "VM removal task failed"); + assert!(error + .downcast_ref::() + .unwrap() + .is_panic()); + assert!(workdir.is_removing()); + assert!(app.refuse_if_removing("vm-1").is_ok()); + assert!(app.launch_lock_handle("vm-1").try_lock().is_ok()); + assert!(app.lock().get("vm-1").is_some()); + assert!(app + .lock() + .cid_pool + .occupy(app.config.cvm.cid_start) + .is_err()); + + app.before_remove_workdir = None; + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + app.finish_remove_vm(mark, false).await.unwrap(); + assert!(!workdir.path().exists()); + assert!(app.lock().get("vm-1").is_none()); + server.abort(); + } + /// A removal keeps retrying while netd is away, keeps the VM marked as /// being removed so nothing can start it, and finishes on its own once netd /// comes back -- without a VMM restart or another request. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 49a797921..7bc380123 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -1003,8 +1003,10 @@ impl VmmRpc for RpcHandler { .context("Failed to load VM") { // Never started, so netd holds nothing for it. - if let Err(err) = fs::remove_dir_all(&work_dir) { - warn!("Failed to remove work dir: {}", err); + match tokio::task::spawn_blocking(move || fs::remove_dir_all(&work_dir)).await { + Ok(Ok(())) => {} + Ok(Err(err)) => warn!("failed to remove work dir: {err:#}"), + Err(err) => warn!("work directory cleanup task failed: {err:#}"), } return Err(err); } @@ -1455,9 +1457,13 @@ impl VmmRpc for RpcHandler { bail!("image '{}' not found", name); } - fs_err::remove_dir_all(&image_dir).with_context(|| { - format!("failed to delete image directory: {}", image_dir.display()) - })?; + tokio::task::spawn_blocking(move || { + fs_err::remove_dir_all(&image_dir).with_context(|| { + format!("failed to delete image directory: {}", image_dir.display()) + }) + }) + .await + .context("image deletion task failed")??; info!("deleted local image: {name}"); Ok(()) @@ -1513,6 +1519,87 @@ mod tests { } } + fn deletion_test_app(path: &Path) -> App { + let mut config: crate::config::Config = + Figment::from(crate::config::load_config_figment(None)) + .extract() + .unwrap(); + config.run_path = path.join("vms"); + config.image.path = path.join("images"); + fs::create_dir_all(&config.run_path).unwrap(); + fs::create_dir_all(&config.image.path).unwrap(); + App::new( + config, + supervisor_client::SupervisorClient::new("http://127.0.0.1:0"), + ) + } + + /// Invariant: failed loading cleans up the new VM's workdir and returns the + /// original load error rather than a cleanup-task result. + #[tokio::test] + async fn create_vm_load_failure_removes_workdir() { + let dir = tempfile::tempdir().unwrap(); + let app = deletion_test_app(dir.path()); + let mut request = test_vm_configuration(); + request.stopped = true; + let error = RpcHandler { app: app.clone() } + .create_vm(request) + .await + .unwrap_err(); + assert_eq!(error.to_string(), "Failed to load VM"); + assert!(format!("{error:#}").contains("Failed to load image")); + assert_eq!(fs::read_dir(app.vm_dir()).unwrap().count(), 0); + assert_eq!(app.lock().iter_vms().count(), 0); + } + + /// Invariant: image deletion removes the selected tree only, reports a + /// filesystem failure with its path, and retains name/missing-image checks. + #[tokio::test] + async fn delete_image_removes_selected_tree_and_preserves_errors() { + let dir = tempfile::tempdir().unwrap(); + let app = deletion_test_app(dir.path()); + let image = app.config.image.path.join("test-image"); + let other = app.config.image.path.join("other-image"); + fs::create_dir_all(image.join("nested")).unwrap(); + fs::write(image.join("nested/rootfs.img"), b"image contents").unwrap(); + fs::write(&other, b"untouched").unwrap(); + RpcHandler { app: app.clone() } + .delete_image(Id { + id: "test-image".into(), + }) + .await + .unwrap(); + assert!(!image.exists()); + assert_eq!(fs::read(&other).unwrap(), b"untouched"); + + let error = RpcHandler { app: app.clone() } + .delete_image(Id { + id: "test-image".into(), + }) + .await + .unwrap_err(); + assert_eq!(error.to_string(), "image 'test-image' not found"); + let error = RpcHandler { app: app.clone() } + .delete_image(Id { + id: "../other-image".into(), + }) + .await + .unwrap_err(); + assert_eq!(error.to_string(), "invalid image name"); + let error = RpcHandler { app } + .delete_image(Id { + id: "other-image".into(), + }) + .await + .unwrap_err(); + assert_eq!( + error.to_string(), + format!("failed to delete image directory: {}", other.display()) + ); + assert!(error.downcast_ref::().is_some()); + assert_eq!(fs::read(&other).unwrap(), b"untouched"); + } + fn pinned(nic_index: Option) -> PortMapping { PortMapping { address: "0.0.0.0".parse().unwrap(), From 2ba129edaf3b8685d5860adc7e00e539ba27bd20 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Sat, 17 Oct 2026 18:34:47 +0800 Subject: [PATCH 2/2] test(vmm): cover orphan retention and marker deletion order Signed-off-by: Leechael Yim --- dstack/vmm/src/app.rs | 56 +++++++++++++++++++++++++++++------ dstack/vmm/src/app/workdir.rs | 32 ++++++++++++++++++++ 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 1d2f06905..9b0396513 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -2395,6 +2395,7 @@ mod tests { /// A supervisor with no remaining process lets removal exercise the real /// directory and netd paths without launching a VM. async fn stopped_supervisor() -> (SupervisorClient, tokio::task::JoinHandle<()>) { + use supervisor_client::supervisor::{ProcessInfo, Response}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); @@ -2413,14 +2414,9 @@ mod tests { } } let body = if is_list { - serde_json::to_string(&supervisor_client::supervisor::Response::Data(Vec::< - supervisor_client::supervisor::ProcessInfo, - >::new( - ))) + serde_json::to_string(&Response::Data(Vec::::new())) } else { - serde_json::to_string(&supervisor_client::supervisor::Response::Data( - Option::::None, - )) + serde_json::to_string(&Response::Data(Option::::None)) } .unwrap(); stream @@ -2495,7 +2491,10 @@ mod tests { workdir.mark_network_cleanup_pending().unwrap(); app.remove_vm("vm-1").await.unwrap(); - gate.entered.await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), gate.entered) + .await + .expect("deletion did not reach the filesystem boundary") + .unwrap(); // The only async worker must be able to run another task while the // filesystem operation is still paused on a different OS thread. tokio::spawn(async { tokio::task::yield_now().await }) @@ -2529,6 +2528,9 @@ mod tests { .occupy(app.config.cvm.cid_start) .is_err()); assert_eq!(netd.methods(), vec!["RemoveVm"]); + let sweep = &netd.seen()[0]; + assert_eq!(sweep["instance_id"], "test-instance"); + assert_eq!(sweep["vm_id"], "vm-1"); gate.release.send(()).unwrap(); let _finished = tokio::time::timeout(Duration::from_secs(5), app.launch_lock("vm-1")) @@ -2575,7 +2577,10 @@ mod tests { let app = app.clone(); async move { app.finish_remove_vm(mark, true).await } }); - gate.entered.await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), gate.entered) + .await + .expect("deletion did not reach the filesystem boundary") + .unwrap(); removal.abort(); assert!(removal.await.unwrap_err().is_cancelled()); assert!(!gate.expired.load(std::sync::atomic::Ordering::SeqCst)); @@ -2603,6 +2608,39 @@ mod tests { server.abort(); } + /// Invariant: orphan cleanup without a persisted removal request must not + /// erase the VM's data, but still finishes state/CID/guard cleanup. + #[tokio::test] + async fn an_unmarked_orphan_keeps_its_workdir() { + let (supervisor, server) = stopped_supervisor().await; + let dir = tempfile::tempdir().unwrap(); + let app = App::new( + test_config(&dir.path().join("absent-netd.sock"), dir.path()), + supervisor, + ); + let workdir = removable_vm(&app, "vm-1"); + assert!(!workdir.is_removing()); + assert!(app.lock().start_removing("vm-1")); + let mark = RemovalMark::new(app.clone(), "vm-1"); + app.finish_remove_vm(mark, false).await.unwrap(); + + assert_eq!( + std::fs::read(workdir.hda_path()).unwrap(), + b"test disk contents" + ); + assert_eq!( + std::fs::read(workdir.shared_dir().join("config")).unwrap(), + b"test config" + ); + assert!(!workdir.is_removing()); + assert!(app.refuse_if_removing("vm-1").is_ok()); + assert!(app.launch_lock_handle("vm-1").try_lock().is_ok()); + let mut state = app.lock(); + assert!(state.get("vm-1").is_none()); + assert_eq!(state.cid_pool.allocate(), Some(app.config.cvm.cid_start)); + server.abort(); + } + /// Invariant: an ordinary deletion failure retains `.removing` for reload, /// while preserving the existing behavior of removing the VM from memory. #[tokio::test] diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index fd67072f3..96771e044 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -375,6 +375,38 @@ mod tests { Ok(()) } + /// Invariant: even a malformed marker is handled after the payloads. A + /// directory in place of the marker produces a real filesystem error, + /// independent of privileges or directory enumeration order. + #[test] + fn remove_all_keeps_a_malformed_marker_until_payloads_are_deleted() { + let dir = tempfile::tempdir().unwrap(); + let workdir = VmWorkDir::new(dir.path().join("vm")); + fs::create_dir_all(workdir.shared_dir()).unwrap(); + fs::write(workdir.hda_path(), b"disk contents").unwrap(); + fs::write(workdir.shared_dir().join("config"), b"config").unwrap(); + fs::create_dir(workdir.removing_marker()).unwrap(); + fs::write(workdir.removing_marker().join("sentinel"), b"marker").unwrap(); + + let error = workdir.remove_all().unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::IsADirectory + ); + assert!(!workdir.hda_path().exists()); + assert!(!workdir.shared_dir().exists()); + assert!(workdir.is_removing()); + assert_eq!( + fs::read(workdir.removing_marker().join("sentinel")).unwrap(), + b"marker" + ); + + fs::remove_dir_all(workdir.removing_marker()).unwrap(); + workdir.set_removing().unwrap(); + workdir.remove_all().unwrap(); + assert!(!workdir.path().exists()); + } + /// `.removing` goes last, so an interrupted deletion is still a removal. #[test] fn remove_all_deletes_everything_and_tolerates_a_missing_directory() {