From 0d890910e204d498fedd886bd826a72cf25d35a0 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 9 Oct 2026 07:09:23 -0700 Subject: [PATCH 1/3] Define result of scalar allreduce and gather helpers in non-MPI builds (cherry picked from commit c9478e6da5e1a44ea02aaf5a2170e68a0a6859ec) --- src/common/m_mpi_common.fpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/common/m_mpi_common.fpp b/src/common/m_mpi_common.fpp index c54568d27..1ba675345 100644 --- a/src/common/m_mpi_common.fpp +++ b/src/common/m_mpi_common.fpp @@ -265,6 +265,8 @@ contains allocate (gathered_vector(sum(recounts))) call MPI_GATHERV(my_vector, counts, mpi_p, gathered_vector, recounts, displs, mpi_p, root, MPI_COMM_WORLD, ierr) +#else + gathered_vector = my_vector #endif end subroutine s_mpi_gather_data @@ -373,6 +375,8 @@ contains integer :: ierr !< Generic flag used to identify and report MPI errors call MPI_ALLREDUCE(var_loc, var_glb, 1, mpi_p, MPI_SUM, MPI_COMM_WORLD, ierr) +#else + var_glb = var_loc #endif end subroutine s_mpi_allreduce_sum @@ -440,6 +444,8 @@ contains integer :: ierr !< Generic flag used to identify and report MPI errors call MPI_ALLREDUCE(var_loc, var_glb, 1, mpi_p, MPI_MIN, MPI_COMM_WORLD, ierr) +#else + var_glb = var_loc #endif end subroutine s_mpi_allreduce_min @@ -470,6 +476,8 @@ contains integer :: ierr !< Generic flag used to identify and report MPI errors call MPI_ALLREDUCE(var_loc, var_glb, 1, mpi_p, MPI_MAX, MPI_COMM_WORLD, ierr) +#else + var_glb = var_loc #endif end subroutine s_mpi_allreduce_max From 1a45dba32f7e7448a8c63823d54ae5e264299edc Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 9 Oct 2026 23:30:58 -0400 Subject: [PATCH 2/3] Default and broadcast particle_cloud shell_axis in pre_process pre_process reads particle_cloud(i)%shell_axis in s_sample_cloud_candidate but never gave it a default or broadcast it; only simulation, which does not use it, did both. Under file_per_process every rank samples the cloud itself, so non-root ranks placed hemisphere-shell particles with an uninitialized axis, and without a namelist value even rank 0 read garbage. Default it to 3 (+z) and broadcast it with the other hand-written particle_cloud members, mirroring simulation. Verified with a 3D hemisphere-shell cloud (shell_axis = 1, file_per_process, 20 particles): the IB state from 2 ranks now matches the 1-rank run exactly and every particle lies on the +x side. (cherry picked from commit 0a3d2d712f714f0cf1c5f59b80a8ff450da65cb5) --- src/pre_process/m_global_parameters.fpp | 1 + src/pre_process/m_mpi_proxy.fpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/pre_process/m_global_parameters.fpp b/src/pre_process/m_global_parameters.fpp index b358516fb..130a25a0b 100644 --- a/src/pre_process/m_global_parameters.fpp +++ b/src/pre_process/m_global_parameters.fpp @@ -396,6 +396,7 @@ contains particle_cloud(i)%moving_ibm = 0 particle_cloud(i)%seed = 0 particle_cloud(i)%cloud_geometry = 1 + particle_cloud(i)%shell_axis = 3 particle_cloud(i)%packing_method = dflt_int particle_cloud(i)%periodic = 0 end do diff --git a/src/pre_process/m_mpi_proxy.fpp b/src/pre_process/m_mpi_proxy.fpp index 6753d818c..592b2995b 100644 --- a/src/pre_process/m_mpi_proxy.fpp +++ b/src/pre_process/m_mpi_proxy.fpp @@ -147,6 +147,7 @@ contains call MPI_BCAST(particle_cloud(i)%moving_ibm, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(particle_cloud(i)%seed, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(particle_cloud(i)%cloud_geometry, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(particle_cloud(i)%shell_axis, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(particle_cloud(i)%packing_method, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(particle_cloud(i)%periodic, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) end do From 1cf902b7e6c1c0710fe43a2371d7707a73c667ed Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 10 Oct 2026 00:20:52 -0400 Subject: [PATCH 3/3] Broadcast periodic_bc to all simulation ranks periodic_bc is set from the global bc_[xyz] values in s_read_input_file, which only rank 0 runs, and was never broadcast. s_mpi_sendrecv_particles reads it on every rank to decide whether to wrap Lagrangian bubble positions across periodic boundaries, so on every other rank it stayed .false. and bubbles leaving through a periodic face were never wrapped. Broadcast it next to bc_io. Computing it after the broadcast is not an option because domain decomposition overwrites the interior ranks' bc values with neighbor ids. pre_process and post_process have no periodic_bc. Verified on a 2-rank 2D Lagrangian bubble case with periodic x: rank 1 now holds periodic_bc = T F F (it held F F F before). Lagrange bubble tests pass. (cherry picked from commit 1c8658469e2a28c31d038a1d648c7928be2aad7f) --- src/simulation/m_mpi_proxy.fpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index 71a13a139..8523bd338 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -149,9 +149,10 @@ contains #:endfor end do - ! manual: cfl_dt (runtime-computed logical), bc_io (BC-file existence) + ! manual: cfl_dt (runtime-computed logical), bc_io (BC-file existence), periodic_bc (from global BCs) call MPI_BCAST(cfl_dt, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(bc_io, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(periodic_bc, 3, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) ! manual: shear_stress, bulk_stress (derived from Re_size post-init on all ranks), ! bodyForces (derived from bf_x/y/z)