From 8cca69ddfa1677a8e92bed5801e218ea406f004d Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 13 Oct 2026 14:36:45 -0700 Subject: [PATCH] Let dua-core queue more of the walk in place of the draining thread dua-core 4.2.0 has `Options::queued_batches_per_worker`, from Byron/dua-cli#413. With room for 128 batches per worker, the walk's loop in a large Rails app waited a median 64 ms on dua-core, against 113 ms with the thread and 301 ms with dua-core's default of 2. --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/packs/walk_directory.rs | 25 ++++++------------------- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3ec20a..0dab722 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -455,9 +455,9 @@ dependencies = [ [[package]] name = "dua-core" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0664627ab16c082ac5136a232dfef44ce0627a6cc43fca6f98e0284d3f9b1d79" +checksum = "d687862f3206e7c660daeb6ab0200c65e58fccd7e508819e4bd9949304c640a8" dependencies = [ "crossbeam", "libc", diff --git a/Cargo.toml b/Cargo.toml index fd73eb6..e8dc54c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ anyhow = { version = "1.0.75", features = [] } # for err clap = { version = "4.2.1", features = ["derive"] } # cli clap_derive = "4.2.0" # cli csv = "1.3.0" # csv de/serialize -dua-core = "4.1.0" # for walking the file tree +dua-core = "4.2.0" # for walking the file tree itertools = "0.15.0" # tools for iterating over iterable things path-clean = "1.0.1" # Pathname#cleaname in Ruby rayon = "1.10.0" # for parallel iteration diff --git a/src/packs/walk_directory.rs b/src/packs/walk_directory.rs index 8be8202..5cbc76b 100644 --- a/src/packs/walk_directory.rs +++ b/src/packs/walk_directory.rs @@ -215,28 +215,19 @@ pub(crate) fn walk_directory( !excluded_dirs_ref.is_match(relative_path) }; - let dua_walk = dua_core::walk( + let walk = dua_core::walk( &absolute_root, // As many threads as rayon's, so RAYON_NUM_THREADS limits the walk too rayon::current_num_threads(), Order::Completion, - Options::default().skip_metadata(), + // With dua-core's default of 2, the workers stop each time this loop + // parses a package.yml (https://github.com/Byron/dua-cli/pull/413) + Options::default() + .skip_metadata() + .queued_batches_per_worker(128), read_directory, ); - // dua-core's workers stop while a few batches wait for this loop, as they - // do each time it parses a package.yml, so this thread drains them. Once a - // release has `Options::queued_batches_per_worker`, set that instead - // (https://github.com/Byron/dua-cli/pull/413). - let (sender, walk) = std::sync::mpsc::channel(); - let draining = std::thread::spawn(move || { - for entry in dua_walk { - if sender.send(entry).is_err() { - break; - } - } - }); - // A file can come before the directory it's in, and before the // package.yml there, so which package.yml each file belongs to is // settled once the walk is done @@ -337,10 +328,6 @@ pub(crate) fn walk_directory( } } - if let Err(panic) = draining.join() { - std::panic::resume_unwind(panic); - } - let package_ymls = package_ymls_of_directories(directories, &package_ymls_in_directories); let owning_package_yml_for_file = included_files_in_directories