Address leftover review comments from merged PRs

.github/scripts/preflight.sh

@@ -151,7 +151,8 @@ run_probe() {
 # reasons -- no input file, no restart data, a missing module -- and none of them
 # say anything about the node. Treating any non-zero status as a fault here would
 # exclude every healthy node in the cluster. 132 is 128+4, a child killed by
-# SIGILL; bash reports signals that way, and mpirun/srun forward it.
+# SIGILL; bash reports signals that way, and mpirun/srun forward it. Some
+# launchers do not pass 132 on, so the "Illegal instruction" text is matched too.
 isa_probe() {
     isa_bin=$(newest_for_device pre_process)
     [ -n "$isa_bin" ] || return 0

.github/scripts/submit-slurm-job.sh

@@ -322,7 +322,7 @@ while :; do
         # 34183404644 exactly this way. sacct knows the node whether or not the
         # .out exists, so identification no longer depends on the marker.
         if [ -z "$faulted_node" ]; then
-            faulted_node=$(sacct -j "$job_id" -X -n -o NodeList 2>/dev/null | head -n1 | tr -d ' ')
+            faulted_node=$(sacct -j "$job_id" -X -n -o NodeList 2>/dev/null | head -n1 | tr -d ' ' || true)
             case "$faulted_node" in ""|None*|*[,\[]*) faulted_node="" ;; esac
         fi
         if [ "$node_attempt" -lt "$MFC_MAX_NODE_RESUBMITS" ]; then

CMakeLists.txt

@@ -116,7 +116,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
     list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/cce")
 endif()
 
-if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang" AND MFC_OpenMP)
     # Otherwise the GPU targets are never up to date and every build redoes the
     # whole-image device LTO link -- ~20 minutes on Frontier, paid on each mfc.sh
     # invocation even when nothing changed. CMake asks the linker for its
@@ -133,7 +133,8 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
     # Directory scope, not a target property: LINK_DEPENDS_USE_LINKER exists only as
     # a variable, and set_target_properties would silently create a custom property
     # that does nothing. Setting it inside MFC_SETUP_TARGET would be function-scoped
-    # and equally silent.
+    # and equally silent. MFC_OpenMP is the amdflang offload build; CPU builds keep
+    # the linker's dependency tracking.
     set(CMAKE_LINK_DEPENDS_USE_LINKER OFF)
 endif()
 

docs/documentation/case.md

@@ -1405,7 +1405,7 @@ The entries labeled "Characteristic." are characteristic boundary conditions bas
 
 This boundary condition can be used for subsonic inflow (`bc_[x,y,z]%[beg,end]` = -7) and subsonic outflow (`bc_[x,y,z]%[beg,end]` = -8) characteristic boundary conditions. These are based on \cite Pirozzoli13. This enables to provide inflow and outflow conditions outside the computational domain.
 
-`bc_[x,y,z]%%vel_in_ramp` starts the inflow smoothly instead of holding it constant, which is what a jet or a tunnel accelerating from rest requires: the start-up is the event of interest, not a transient to be discarded. The inflow velocity is scaled by \f$f(t) = f_0 + (1 - f_0)\left[1 + \tanh\left(6 (t - t_0)/\tau - 3\right)\right]/2\f$, with \f$\tau\f$ = `vel_in_ramp`, \f$t_0\f$ = `vel_in_t0` and \f$f_0\f$ = `vel_in_frac0`, so it leaves \f$f_0\f$ of the final velocity at \f$t_0\f$ and is within half a percent of it at \f$t_0 + \tau\f$. A boundary with `vel_in_ramp = 0` is held constant, as before. The ramp acts on a GRCBC inflow (`grcbc_in`) or on the ghost-cell velocity of a Dirichlet (-17) boundary or boundary patch normal to that direction, such as a nozzle set into a wall.
+`bc_[x,y,z]%%vel_in_ramp` starts the inflow smoothly instead of holding it constant, which is what a jet or a tunnel accelerating from rest requires: the start-up is the event of interest, not a transient to be discarded. The inflow velocity is scaled by \f$f(t) = f_0 + (1 - f_0)\left[1 + \tanh\left(6 (t - t_0)/\tau - 3\right)\right]/2\f$, with \f$\tau\f$ = `vel_in_ramp`, \f$t_0\f$ = `vel_in_t0` and \f$f_0\f$ = `vel_in_frac0`, so it leaves \f$\approx f_0\f$ of the final velocity at \f$t_0\f$ and is within half a percent of it at \f$t_0 + \tau\f$. A boundary with `vel_in_ramp = 0` is held constant, as before. The ramp acts on a GRCBC inflow (`grcbc_in`) or on the ghost-cell velocity of a Dirichlet (-17) boundary or boundary patch normal to that direction, such as a nozzle set into a wall.
 
 ### Patch types {#patch-types}
 

examples/2D_ibm_thin_plate_force/case.py

@@ -33,14 +33,11 @@
 t_end = t0 + t_p + 4.0
 x0, x1, y0, y1 = -2.0, 5.0, -2.5, 2.5
 
-# L4 added after the first three failed to converge: refining 2 -> 4 -> 8 cells across the thickness moved
+# NCELL=16 added after 2, 4 and 8 failed to converge: refining 2 -> 4 -> 8 cells across the thickness moved
 # the peak lift 6.41 -> 4.68 -> 4.46 against a reference of 7.00, i.e. away from it and then stalling.
 # Two under-resolved answers landing near each other is not convergence. If a few cells across a thin
 # body is simply too few for the immersed boundary, 16 should move back toward the reference; if the
 # finite thickness is genuinely the difference, it should stay near 4.5.
-LEVELS = {"L1": 0.0125, "L2": 0.00625, "L3": 0.003125, "L4": 0.0015625}
-
-
 NCELL = int(os.environ.get("NCELL", 4))
 dx = THICK / NCELL
 m, n = int((x1 - x0) / dx) - 1, int((y1 - y0) / dx) - 1

examples/3D_ibm_neighborhood_radius/case.py

@@ -126,7 +126,10 @@
     if os.environ.get("SUMMARY"):
         bound = 0.5 * math.sqrt(c**2 + SPAN**2 + THICK**2)
         print(f"plate half-extent (s_get_ib_bound, geometry 9): {bound:.4f}")
-        print(f"rank extents at 64 ranks (16 x 2 x 2): x {(x1 - x0) / 16:.3f}, " f"y {(y1 - y0) / 2:.3f}, z {(z1 - z0) / 2:.3f}")
-        print(f"hops needed across the thinnest rank: ceil(1.1 * {bound:.4f} / {(x1 - x0) / 16:.3f}) = " f"{max(1, math.ceil(1.1 * bound / ((x1 - x0) / 16)))}")
+        # Ranks per direction, as MFC's topology search picks them for 400 x 50 x 50 at 64 ranks; set TOPOLOGY otherwise.
+        topo = [int(t) for t in os.environ.get("TOPOLOGY", "16,2,2").split(",")]
+        ext = [(hi - lo) / t for (lo, hi), t in zip(((x0, x1), (y0, y1), (z0, z1)), topo)]
+        print(f"rank extents at {topo[0] * topo[1] * topo[2]} ranks ({' x '.join(map(str, topo))}): x {ext[0]:.3f}, y {ext[1]:.3f}, z {ext[2]:.3f}")
+        print(f"hops needed across the thinnest rank: ceil(1.1 * {bound:.4f} / {min(ext):.3f}) = {max(1, math.ceil(1.1 * bound / min(ext)))}")
     else:
         print(json.dumps(case, indent=4))

src/common/m_derived_types.fpp

@@ -113,8 +113,8 @@ module m_derived_types
         real(wp), dimension(num_fluids_max) :: alpha_rho_in, alpha_in
         logical                             :: grcbc_in, grcbc_out, grcbc_vel_out
         !> Smooth start-up of a GRCBC inflow: the inflow velocity is scaled by f(t) = vel_in_frac0 + (1 - vel_in_frac0) (1 + tanh(6
-        !! (t - t0)/tau - 3))/2, so it leaves vel_in_frac0 of its final value at t0 and reaches it after vel_in_ramp. Inactive when
-        !! the ramp duration is zero.
+        !! (t - t0)/tau - 3))/2, so f(t0) ~ vel_in_frac0 and f(t0 + tau) ~ 1 (within 0.25%). Inactive when the ramp duration is
+        !! zero.
         real(wp) :: vel_in_ramp, vel_in_t0, vel_in_frac0
         logical  :: isothermal_in, isothermal_out
         real(wp) :: Twall_in, Twall_out

src/simulation/m_start_up.fpp

@@ -650,12 +650,12 @@ contains
         end if
 
         ! Advance time after RK so source terms see current-step time. With a fixed dt, use the same t_step*dt a restart
-        ! starts from (p_main): a running sum drifts from it (1290 ulps by step 27000), so a restarted run would see the
-        ! prescribed IB kinematics, inflow ramps and forcing at slightly different times than the run it continues
+        ! starts from (p_main): a running sum drifts from it, so a restarted run would see the prescribed IB kinematics,
+        ! inflow ramps and forcing at slightly different times than the run it continues
         if (cfl_dt) then
             mytime = mytime + dt
         else
-            mytime = (t_step + 1)*dt
+            mytime = real(t_step + 1, wp)*dt
         end if
 
         if (relax) call s_infinite_relaxation_k(q_cons_ts(1)%vf)
@@ -704,7 +704,7 @@ contains
                 open (1, file='time_data.dat', position='append', status='old')
             else
                 open (1, file='time_data.dat', status='new')
-                ! time_final is the fastest single RK stage (one RHS evaluation), not a whole step; see s_tvd_rk
+                ! time_final: max over ranks of each rank's fastest RK stage (one RHS evaluation), not a whole step; see s_tvd_rk
                 write (1, '(A10, A15, A15)') "Ranks", "s/rhs", "ns/gp/eq/rhs"
             end if
 
@@ -717,7 +717,7 @@ contains
                 open (1, file='io_time_data.dat', position='append', status='old')
             else
                 open (1, file='io_time_data.dat', status='new')
-                ! io_time_final is the mean time of one s_save_data call
+                ! io_time_final is the max over ranks of each rank's mean s_save_data time
                 write (1, '(A10, A15)') "Ranks", "s/save"
             end if
 

src/simulation/m_time_steppers.fpp

@@ -876,8 +876,8 @@ contains
             if (time_stepper == time_stepper_rk3 .and. s == 2) t_stage = mytime + 0.5_wp*dt
         else
             ! The same t_step*dt form a restart evaluates the kinematics at, so a restart sees bitwise the same body
-            t_stage = (t_step + 1)*dt
-            if (time_stepper == time_stepper_rk3 .and. s == 2) t_stage = (t_step + 0.5_wp)*dt
+            t_stage = real(t_step + 1, wp)*dt
+            if (time_stepper == time_stepper_rk3 .and. s == 2) t_stage = (real(t_step, wp) + 0.5_wp)*dt
         end if
 
         $:GPU_PARALLEL_LOOP(private='[i, gbl_id]', copyin='[s, t_stage]')

toolchain/mfc/case_validator.py

@@ -1073,7 +1073,7 @@ def _eos_coefficient_args(self, i, family):
         args = []
         for suffix in family.coefficients_args:
             value = self.get(f"fluid_pp({i})%{family.prefix}_{suffix}")
-            args.append((value or 0.0) if suffix in optional else value)
+            args.append(0.0 if value is None and suffix in optional else value)
         return args
 
     def _check_initial_states_inside_eos(self, num_fluids):
@@ -1960,10 +1960,10 @@ def check_grcbc(self):
                 # dir_idx, which is (2,1,3) for a y inflow and (3,1,2) for z -- so requiring only
                 # component 1 would leave the normal velocity of a y or z inflow unchecked.
                 num_dims = 3 if (self.get("p", 0) or 0) > 0 else (2 if (self.get("n", 0) or 0) > 0 else 1)
-                missing = [n for n in (f"bc_{dir}%pres_in",) if self.get(n) is None]
-                missing += [f"bc_{dir}%vel_in({d})" for d in range(1, num_dims + 1) if self.get(f"bc_{dir}%vel_in({d})") is None]
-                missing += [f"bc_{dir}%alpha_rho_in({i})" for i in range(1, num_fluids + 1) if self.get(f"bc_{dir}%alpha_rho_in({i})") is None]
-                missing += [f"bc_{dir}%alpha_in({i})" for i in range(1, num_fluids + 1) if self.get(f"bc_{dir}%alpha_in({i})") is None]
+                required = [f"bc_{dir}%pres_in"]
+                required += [f"bc_{dir}%vel_in({d})" for d in range(1, num_dims + 1)]
+                required += [f"bc_{dir}%{name}({i})" for name in ("alpha_rho_in", "alpha_in") for i in range(1, num_fluids + 1)]
+                missing = [name for name in required if self.get(name) is None]
                 self.prohibit(len(missing) > 0, f"Subsonic Inflow (grcbc_in) needs the full inflow state; missing {', '.join(missing)}")
             if grcbc_out:
                 # Check if EITHER beg OR end is set to -8

toolchain/mfc/test_thermochem.py

@@ -67,14 +67,24 @@
         FPE_TRAP: ["-ffpe-trap=invalid,zero,overflow"],
     },
     # CCE enables OpenACC by default; turn it off unless asked for, as cmake/MFCTargets.cmake does.
-    "Cray": {"base": ["-eZ"], None: ["-hnoacc"], "acc": ["-hacc"], "mp": ["-hnoacc", "-fopenmp"], FPE_TRAP: ["-Ktrap=divz,inv,ovf"]},
+    "Cray": {
+        "base": ["-eZ"],
+        None: ["-hnoacc"],
+        "acc": ["-hacc"],
+        "mp": ["-hnoacc", "-fopenmp"],
+        FPE_TRAP: ["-Ktrap=divz,inv,ovf"],
+    },
 }
 
 
-@functools.cache
 def fortran_compiler():
     """The compiler MFC's build would use ($FC, else ftn or gfortran on PATH) and its family, or None."""
-    names = [os.environ["FC"]] if os.environ.get("FC") else ["ftn", "gfortran"]
+    return _probe_compiler(os.environ.get("FC") or None)
+
+
+@functools.cache  # keyed on $FC, so a changed $FC is probed afresh
+def _probe_compiler(fc):
+    names = [fc] if fc else ["ftn", "gfortran"]
     for name in names:
         path = shutil.which(name)
         if path is None: