diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index b1a3d3fca..d604be894 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -170,7 +170,14 @@ def create_directory(dirpath: str) -> None: def delete_directory(dirpath: str) -> None: if os.path.isdir(dirpath): - shutil.rmtree(dirpath) + try: + shutil.rmtree(dirpath) + except FileNotFoundError: + # Under heavy parallel metadata load (Lustre) an rmdir/unlink can report ENOENT for an entry that is already gone. + # Absence is the goal: finish the sweep and only fail if something is left behind. + shutil.rmtree(dirpath, ignore_errors=True) + if os.path.isdir(dirpath): + raise def get_program_output(arguments: typing.List[str] = None, cwd=None, merge_stderr: bool = False): diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 53cfa9348..885b9cc67 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -1,3 +1,4 @@ +import glob import itertools import math import os @@ -831,11 +832,9 @@ def _handle_case(case: TestCase, devices: typing.Set[int]): out_filepath = os.path.join(case.get_dirpath(), "out_post.txt") common.file_write(out_filepath, cmd.stdout) - silo_dir = os.path.join(case.get_dirpath(), "silo_hdf5", "p0") - if os.path.isdir(silo_dir): - for silo_filename in os.listdir(silo_dir): - silo_filepath = os.path.join(silo_dir, silo_filename) - _process_silo_file(silo_filepath, case, out_filepath) + # glob, not isdir+listdir: no window for the directory to vanish in between. + for silo_filepath in glob.glob(os.path.join(case.get_dirpath(), "silo_hdf5", "p0", "*")): + _process_silo_file(silo_filepath, case, out_filepath) case.delete_output() diff --git a/toolchain/mfc/test_common.py b/toolchain/mfc/test_common.py index a5d9e952d..2734269d2 100644 --- a/toolchain/mfc/test_common.py +++ b/toolchain/mfc/test_common.py @@ -1,11 +1,12 @@ import os +import shutil import stat import sys import tempfile from pathlib import Path from unittest.mock import patch -from mfc.common import get_py_program_output +from mfc.common import delete_directory, get_py_program_output def test_case_files_run_under_the_toolchain_interpreter(): @@ -43,3 +44,22 @@ def test_a_hostile_python3_on_path_is_not_consulted(): out, err = get_py_program_output(str(case), []) assert err == 0, "case file ran under PATH's python3 instead of sys.executable" assert out.strip() == "loaded" + + +def test_delete_directory_tolerates_enoent_from_a_concurrent_unlink(): + """rmtree can raise ENOENT mid-sweep for an entry already gone (#1968); finish the sweep instead.""" + real_rmtree = shutil.rmtree + + def flaky_rmtree(path, *args, **kwargs): + if not kwargs.get("ignore_errors"): + os.rmdir(os.path.join(path, "p0")) + raise FileNotFoundError(2, "No such file or directory", "p0") + return real_rmtree(path, *args, **kwargs) + + with tempfile.TemporaryDirectory() as d: + target = Path(d) / "silo_hdf5" + (target / "p0").mkdir(parents=True) + (target / "other").mkdir() + with patch("mfc.common.shutil.rmtree", flaky_rmtree): + delete_directory(str(target)) + assert not target.exists()