Users/gilbelee/fix ibv probe crash

CHANGELOG.md

@@ -3,6 +3,16 @@
 Documentation for TransferBench is available at
 [https://rocm.docs.amd.com/projects/TransferBench](https://rocm.docs.amd.com/projects/TransferBench).
 
+## v1.71.01
+### Modified
+- `a2a` and `a2asweep` presets now default `A2A_DIRECT=0` on MI4XX (gfx1250) hardware, allowing DPX to route
+  transfers more intuitively across the full all-to-all. Other hardware keeps the previous `A2A_DIRECT=1` default
+### Fixed
+- NIC topology detection now probes each IBV device (`ibv_open_device` / `ibv_query_device` / `ibv_query_port`) in a
+  forked child process, so a crash inside a third-party ibverbs provider plugin (observed with a vendor RoCE driver)
+  only disables that one NIC instead of crashing TransferBench on startup. The affected device is reported as
+  inactive with a warning instead
+
 ## v1.71.00
 ### Added
 - Added PingPong operations to measure latency between two GPU Executors. A PingPong joins a Ping and a Pong triplet

CMakeLists.txt

@@ -104,7 +104,7 @@ set(ENV{ROCM_PATH} "${ROCM_PATH}")
 #==================================================================================================
 set(TRANSFERBENCH_VERSION_MAJOR 1)
 set(TRANSFERBENCH_VERSION_MINOR 71)
-set(TRANSFERBENCH_VERSION_PATCH_FALLBACK "00")
+set(TRANSFERBENCH_VERSION_PATCH_FALLBACK "01")
 
 # Auto-compute patch from git: count commits since the last v<MAJOR>.<MINOR>.* tag.
 # Falls back to TRANSFERBENCH_VERSION_PATCH_FALLBACK when git is unavailable,

docs/reference/presets.rst

@@ -124,7 +124,7 @@ To modify the behavior of a2a preset, use the following environment variables:
       - To use only directly connected XGMI links (hop count = 1). 0 = full all-to-all.
 
         This can be useful on older MI2XX hardware that doesn't feature full all-to-all XGMI connectivity. Running the standard all-to-all between all pairs of GPUs on that hardware ends up using XGMI links more than once.
-      - ``1``
+      - ``1`` (``0`` on MI4XX, to let DPX route transfers)
 
     * - ``A2A_LOCAL``
       - To include local transfers (i→i). 0 = exclude, 1 = include.
@@ -323,7 +323,7 @@ To modify the behavior of a2asweep preset, use the following environment variabl
 
     * - ``A2A_DIRECT``
       - To use only directly-connected GPU pairs, set to ``1``. For full all-to-all, set to ``0``.
-      - ``1``
+      - ``1`` (``0`` on MI4XX, to let DPX route transfers)
 
     * - ``A2A_LOCAL``
       - To include local transfers, set to ``1``. To exclude, set to ``0``.

src/client/EnvVars.hpp

@@ -44,7 +44,7 @@ THE SOFTWARE.
 #include <random>
 #include <time.h>
 
-#define CLIENT_VERSION "00"
+#define CLIENT_VERSION "01"
 
 #include "TransferBench.hpp"
 using namespace TransferBench;
@@ -138,28 +138,34 @@ class EnvVars
   // Developer features
   int gpuMaxHwQueues;                // Tracks GPU_MAX_HW_QUEUES environment variable
 
+  // Detected hardware
+  std::string gpuArchName;           // GCN arch name of GPU 0 (e.g. "gfx942"), empty if no GPU detected
+  int defaultA2aDirect;              // Per-arch default for A2A_DIRECT (0 on MI4XX so DPX can route transfers)
+
   // Constructor that collects values
   EnvVars()
   {
     // Try to detect the GPU
     hipDeviceProp_t prop;
     std::string fullName = "";
-    std::string archName = "";
     int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX);
     if (numDetectedGpus > 0) {
       HIP_CALL(hipGetDeviceProperties(&prop, 0));
       fullName = prop.gcnArchName;
-      archName = fullName.substr(0, fullName.find(':'));
+      gpuArchName = fullName.substr(0, fullName.find(':'));
     }
 
     // Different hardware pick different GPU kernels
     // This performance difference is generally only noticable when executing fewer CUs
     int defaultGfxUnroll = 4;
-    if      (archName == "gfx906")  defaultGfxUnroll = 8;
-    else if (archName == "gfx90a")  defaultGfxUnroll = 8;
-    else if (archName == "gfx942")  defaultGfxUnroll = 4;
-    else if (archName == "gfx950")  defaultGfxUnroll = 4;
-    else if (archName == "gfx1250") defaultGfxUnroll = 32;
+    if      (gpuArchName == "gfx906")  defaultGfxUnroll = 8;
+    else if (gpuArchName == "gfx90a")  defaultGfxUnroll = 8;
+    else if (gpuArchName == "gfx942")  defaultGfxUnroll = 4;
+    else if (gpuArchName == "gfx950")  defaultGfxUnroll = 4;
+    else if (gpuArchName == "gfx1250") defaultGfxUnroll = 32;
+
+    // On MI4XX (gfx1250), default to full all-to-all so DPX can route transfers more intuitively
+    defaultA2aDirect = (gpuArchName == "gfx1250") ? 0 : 1;
 
     alwaysValidate     = GetEnvVar("ALWAYS_VALIDATE", 0);
     blockBytes         = GetEnvVar("BLOCK_BYTES"         , 256);

src/client/Presets/AllToAll.hpp

@@ -46,7 +46,7 @@ int AllToAllPreset(EnvVars&          ev,
   int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX);
 
   // Collect env vars for this preset
-  int a2aDirect     = EnvVars::GetEnvVar("A2A_DIRECT"     , 1);
+  int a2aDirect     = EnvVars::GetEnvVar("A2A_DIRECT"     , ev.defaultA2aDirect);
   int a2aLocal      = EnvVars::GetEnvVar("A2A_LOCAL"      , 0);
   int memTypeIdx    = EnvVars::GetEnvVar("MEM_TYPE"       , 2);
   int numGpus       = EnvVars::GetEnvVar("NUM_GPU_DEVICES", numDetectedGpus);

src/client/Presets/AllToAllSweep.hpp

@@ -49,7 +49,7 @@ int AllToAllSweepPreset(EnvVars&          ev,
   int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX);
 
   // Collect env vars for this preset
-  int a2aDirect     = EnvVars::GetEnvVar("A2A_DIRECT"     , 1);
+  int a2aDirect     = EnvVars::GetEnvVar("A2A_DIRECT"     , ev.defaultA2aDirect);
   int a2aLocal      = EnvVars::GetEnvVar("A2A_LOCAL"      , 0);
   int memTypeIdx    = EnvVars::GetEnvVar("MEM_TYPE"       , 2);
   int numGpus       = EnvVars::GetEnvVar("NUM_GPU_DEVICES", numDetectedGpus);

src/header/TransferBench.hpp

@@ -58,6 +58,7 @@ THE SOFTWARE.
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/utsname.h>
+#include <sys/wait.h>
 #include <thread>
 #include <type_traits>
 #include <unistd.h>
@@ -3659,6 +3660,107 @@ const auto& AmdSmiFabricInfoV1(const T& info)
     return ERR_NONE;
   }
 
+  // Plain-old-data summary of the per-device probe below, sized so it can be
+  // shipped across a pipe to/from a forked child with a single read/write.
+  struct IbvProbeResult
+  {
+    bool     hasActivePort = false;
+    bool     isRoce        = false;
+    int      gidIndex      = -1;
+    uint32_t maxMsgSize    = 0;
+    char     gidDescriptor[64] = {};
+  };
+
+  // Opens/queries a single IBV device and fills in `result`. Runs inside the
+  // forked child spawned by ProbeIbvDeviceSafe() below.
+  static void RunIbvDeviceProbe(ibv_device* devicePtr, IbvProbeResult& result)
+  {
+    struct ibv_context *context = ibv_open_device(devicePtr);
+    if (context) {
+      struct ibv_device_attr deviceAttr;
+      if (!ibv_query_device(context, &deviceAttr)) {
+        int activePort = 0;
+        for (int port = 1; port <= deviceAttr.phys_port_cnt; ++port) {
+          struct ibv_port_attr portAttr;
+          if (ibv_query_port(context, port, &portAttr)) continue;
+          if (portAttr.state != IBV_PORT_ACTIVE) continue;
+
+          // Any active port may be selected for QP setup via cfg.nic.ibPort, which is
+          // not visible here, so keep the most restrictive limit across all of them.
+          if (portAttr.max_msg_sz &&
+              (!result.maxMsgSize || portAttr.max_msg_sz < result.maxMsgSize))
+            result.maxMsgSize = portAttr.max_msg_sz;
+
+          // The first active port supplies the RoCE / GID information
+          if (result.hasActivePort) continue;
+          activePort = port;
+          result.hasActivePort = true;
+          if (portAttr.link_layer == IBV_LINK_LAYER_ETHERNET) {
+            result.isRoce = true;
+            std::pair<int, std::string> gidInfo (-1, "");
+            auto res = GetGidIndex(context, portAttr.gid_tbl_len, activePort, gidInfo);
+            if (res.errType == ERR_NONE) {
+              result.gidIndex = gidInfo.first;
+              strncpy(result.gidDescriptor, gidInfo.second.c_str(), sizeof(result.gidDescriptor) - 1);
+            }
+          }
+        }
+      }
+      ibv_close_device(context);
+    }
+  }
+
+  // Probes a single IBV device (ibv_open_device/query_device/query_port) in a
+  // forked child process. Third-party ibverbs provider plugins (e.g. vendor
+  // RoCE drivers) have been observed to crash inside ibv_open_device() for
+  // devices that are listed but whose kernel/userspace driver pairing is
+  // broken; isolating the probe in a child means that crash only costs us
+  // this one device instead of taking down the whole benchmark.
+  static bool ProbeIbvDeviceSafe(ibv_device* devicePtr, IbvProbeResult& result)
+  {
+    result = IbvProbeResult{};
+
+    int fds[2];
+    if (pipe(fds) != 0) return false;
+
+    pid_t pid = fork();
+    if (pid < 0) {
+      close(fds[0]);
+      close(fds[1]);
+      return false;
+    }
+
+    if (pid == 0) {
+      // Child: probe the device and report back over the pipe, then exit
+      // without running parent-side atexit/destructor state.
+      close(fds[0]);
+      IbvProbeResult childResult{};
+      RunIbvDeviceProbe(devicePtr, childResult);
+      ssize_t written = write(fds[1], &childResult, sizeof(childResult));
+      (void)written;
+      close(fds[1]);
+      _exit(0);
+    }
+
+    // Parent
+    close(fds[1]);
+    size_t total = 0;
+    char* buf = reinterpret_cast<char*>(&result);
+    while (total < sizeof(result)) {
+      ssize_t n = read(fds[0], buf + total, sizeof(result) - total);
+      if (n <= 0) break;
+      total += (size_t)n;
+    }
+    close(fds[0]);
+
+    int status = 0;
+    waitpid(pid, &status, 0);
+
+    bool ok = (total == sizeof(result)) && WIFEXITED(status) && WEXITSTATUS(status) == 0;
+    if (!ok) result = IbvProbeResult{};
+    return ok;
+  }
+
   static vector<IbvDevice>& GetIbvDeviceList()
   {
     static bool isInitialized = false;
@@ -3690,39 +3792,17 @@ const auto& AmdSmiFabricInfoV1(const T& info)
           ibvDevice.isRoce = false;
           ibvDevice.gidIndex = -1;
           {
-            struct ibv_context *context = ibv_open_device(ibvDevice.devicePtr);
-            if (context) {
-              struct ibv_device_attr deviceAttr;
-              if (!ibv_query_device(context, &deviceAttr)) {
-                int activePort;
-                for (int port = 1; port <= deviceAttr.phys_port_cnt; ++port) {
-                  struct ibv_port_attr portAttr;
-                  if (ibv_query_port(context, port, &portAttr)) continue;
-                  if (portAttr.state != IBV_PORT_ACTIVE) continue;
-
-                  // Any active port may be selected for QP setup via cfg.nic.ibPort, which is
-                  // not visible here, so keep the most restrictive limit across all of them.
-                  if (portAttr.max_msg_sz &&
-                      (!ibvDevice.maxMsgSize || portAttr.max_msg_sz < ibvDevice.maxMsgSize))
-                    ibvDevice.maxMsgSize = portAttr.max_msg_sz;
-
-                  // The first active port supplies the RoCE / GID information
-                  if (ibvDevice.hasActivePort) continue;
-                  activePort = port;
-                  ibvDevice.hasActivePort = true;
-                  if (portAttr.link_layer == IBV_LINK_LAYER_ETHERNET) {
-                    ibvDevice.isRoce = true;
-                    std::pair<int, std::string> gidInfo (-1, "");
-                    auto res = GetGidIndex(context, portAttr.gid_tbl_len, activePort, gidInfo);
-                    if (res.errType == ERR_NONE) {
-                      ibvDevice.gidIndex = gidInfo.first;
-                      ibvDevice.gidDescriptor = gidInfo.second;
-                    }
-                  }
-                }
-              }
-              ibv_close_device(context);
+            IbvProbeResult probeResult;
+            if (!ProbeIbvDeviceSafe(ibvDevice.devicePtr, probeResult)) {
+              fprintf(stderr, "[TransferBench] Warning: probing NIC device '%s' crashed or failed; "
+                      "treating it as inactive. Use TB_NIC_FILTER to exclude it from future runs.\n",
+                      ibvDevice.name.c_str());
             }
+            ibvDevice.hasActivePort = probeResult.hasActivePort;
+            ibvDevice.isRoce        = probeResult.isRoce;
+            ibvDevice.gidIndex      = probeResult.gidIndex;
+            ibvDevice.maxMsgSize    = probeResult.maxMsgSize;
+            ibvDevice.gidDescriptor = probeResult.gidDescriptor;
           }
           ibvDevice.busId = "";
           {