Fix flaky p0 ENOENT in test harness directory handling (#1968)

toolchain/mfc/common.py

@@ -170,7 +170,14 @@ def create_directory(dirpath: str) -> None:
 
 def delete_directory(dirpath: str) -> None:
     if os.path.isdir(dirpath):
-        shutil.rmtree(dirpath)
+        try:
+            shutil.rmtree(dirpath)
+        except FileNotFoundError:
+            # Under heavy parallel metadata load (Lustre) an rmdir/unlink can report ENOENT for an entry that is already gone.
+            # Absence is the goal: finish the sweep and only fail if something is left behind.
+            shutil.rmtree(dirpath, ignore_errors=True)
+            if os.path.isdir(dirpath):
+                raise
 
 
 def get_program_output(arguments: typing.List[str] = None, cwd=None, merge_stderr: bool = False):

toolchain/mfc/test/test.py

@@ -1,3 +1,4 @@
+import glob
 import itertools
 import math
 import os
@@ -831,11 +832,9 @@ def _handle_case(case: TestCase, devices: typing.Set[int]):
             out_filepath = os.path.join(case.get_dirpath(), "out_post.txt")
             common.file_write(out_filepath, cmd.stdout)
 
-            silo_dir = os.path.join(case.get_dirpath(), "silo_hdf5", "p0")
-            if os.path.isdir(silo_dir):
-                for silo_filename in os.listdir(silo_dir):
-                    silo_filepath = os.path.join(silo_dir, silo_filename)
-                    _process_silo_file(silo_filepath, case, out_filepath)
+            # glob, not isdir+listdir: no window for the directory to vanish in between.
+            for silo_filepath in glob.glob(os.path.join(case.get_dirpath(), "silo_hdf5", "p0", "*")):
+                _process_silo_file(silo_filepath, case, out_filepath)
 
         case.delete_output()
 

toolchain/mfc/test_common.py

@@ -1,11 +1,12 @@
 import os
+import shutil
 import stat
 import sys
 import tempfile
 from pathlib import Path
 from unittest.mock import patch
 
-from mfc.common import get_py_program_output
+from mfc.common import delete_directory, get_py_program_output
 
 
 def test_case_files_run_under_the_toolchain_interpreter():
@@ -43,3 +44,22 @@ def test_a_hostile_python3_on_path_is_not_consulted():
             out, err = get_py_program_output(str(case), [])
     assert err == 0, "case file ran under PATH's python3 instead of sys.executable"
     assert out.strip() == "loaded"
+
+
+def test_delete_directory_tolerates_enoent_from_a_concurrent_unlink():
+    """rmtree can raise ENOENT mid-sweep for an entry already gone (#1968); finish the sweep instead."""
+    real_rmtree = shutil.rmtree
+
+    def flaky_rmtree(path, *args, **kwargs):
+        if not kwargs.get("ignore_errors"):
+            os.rmdir(os.path.join(path, "p0"))
+            raise FileNotFoundError(2, "No such file or directory", "p0")
+        return real_rmtree(path, *args, **kwargs)
+
+    with tempfile.TemporaryDirectory() as d:
+        target = Path(d) / "silo_hdf5"
+        (target / "p0").mkdir(parents=True)
+        (target / "other").mkdir()
+        with patch("mfc.common.shutil.rmtree", flaky_rmtree):
+            delete_directory(str(target))
+        assert not target.exists()