From 9a53e5bb0cc29af67f61a625dcda0c384fbdf2c4 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 9 Oct 2026 08:56:26 -0700 Subject: [PATCH 1/2] Tolerate ENOENT while deleting test output directories (#1968) --- toolchain/mfc/common.py | 9 ++++++++- toolchain/mfc/test_common.py | 22 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index b1a3d3fcac..d604be894a 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -170,7 +170,14 @@ def create_directory(dirpath: str) -> None: def delete_directory(dirpath: str) -> None: if os.path.isdir(dirpath): - shutil.rmtree(dirpath) + try: + shutil.rmtree(dirpath) + except FileNotFoundError: + # Under heavy parallel metadata load (Lustre) an rmdir/unlink can report ENOENT for an entry that is already gone. + # Absence is the goal: finish the sweep and only fail if something is left behind. + shutil.rmtree(dirpath, ignore_errors=True) + if os.path.isdir(dirpath): + raise def get_program_output(arguments: typing.List[str] = None, cwd=None, merge_stderr: bool = False): diff --git a/toolchain/mfc/test_common.py b/toolchain/mfc/test_common.py index a5d9e952d4..2734269d28 100644 --- a/toolchain/mfc/test_common.py +++ b/toolchain/mfc/test_common.py @@ -1,11 +1,12 @@ import os +import shutil import stat import sys import tempfile from pathlib import Path from unittest.mock import patch -from mfc.common import get_py_program_output +from mfc.common import delete_directory, get_py_program_output def test_case_files_run_under_the_toolchain_interpreter(): @@ -43,3 +44,22 @@ def test_a_hostile_python3_on_path_is_not_consulted(): out, err = get_py_program_output(str(case), []) assert err == 0, "case file ran under PATH's python3 instead of sys.executable" assert out.strip() == "loaded" + + +def test_delete_directory_tolerates_enoent_from_a_concurrent_unlink(): + """rmtree can raise ENOENT mid-sweep for an entry already gone (#1968); finish the sweep instead.""" + real_rmtree = shutil.rmtree + + def flaky_rmtree(path, *args, **kwargs): + if not kwargs.get("ignore_errors"): + os.rmdir(os.path.join(path, "p0")) + raise FileNotFoundError(2, "No such file or directory", "p0") + return real_rmtree(path, *args, **kwargs) + + with tempfile.TemporaryDirectory() as d: + target = Path(d) / "silo_hdf5" + (target / "p0").mkdir(parents=True) + (target / "other").mkdir() + with patch("mfc.common.shutil.rmtree", flaky_rmtree): + delete_directory(str(target)) + assert not target.exists() From 0302e9981b80fd0f9d1a117bc1e78c193f6406e1 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 9 Oct 2026 09:27:51 -0700 Subject: [PATCH 2/2] Scan silo p0 output with glob instead of isdir+listdir (#1968) --- toolchain/mfc/test/test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 53cfa93488..885b9cc678 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -1,3 +1,4 @@ +import glob import itertools import math import os @@ -831,11 +832,9 @@ def _handle_case(case: TestCase, devices: typing.Set[int]): out_filepath = os.path.join(case.get_dirpath(), "out_post.txt") common.file_write(out_filepath, cmd.stdout) - silo_dir = os.path.join(case.get_dirpath(), "silo_hdf5", "p0") - if os.path.isdir(silo_dir): - for silo_filename in os.listdir(silo_dir): - silo_filepath = os.path.join(silo_dir, silo_filename) - _process_silo_file(silo_filepath, case, out_filepath) + # glob, not isdir+listdir: no window for the directory to vanish in between. + for silo_filepath in glob.glob(os.path.join(case.get_dirpath(), "silo_hdf5", "p0", "*")): + _process_silo_file(silo_filepath, case, out_filepath) case.delete_output()