diff --git a/CHANGELOG.md b/CHANGELOG.md index da085248..64adaa3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ Documentation for TransferBench is available at [https://rocm.docs.amd.com/projects/TransferBench](https://rocm.docs.amd.com/projects/TransferBench). +## v1.71.01 +### Modified +- `a2a` and `a2asweep` presets now default `A2A_DIRECT=0` on MI4XX (gfx1250) hardware, allowing DPX to route + transfers more intuitively across the full all-to-all. Other hardware keeps the previous `A2A_DIRECT=1` default +### Fixed +- NIC topology detection now probes each IBV device (`ibv_open_device` / `ibv_query_device` / `ibv_query_port`) in a + forked child process, so a crash inside a third-party ibverbs provider plugin (observed with a vendor RoCE driver) + only disables that one NIC instead of crashing TransferBench on startup. The affected device is reported as + inactive with a warning instead + ## v1.71.00 ### Added - Added PingPong operations to measure latency between two GPU Executors. A PingPong joins a Ping and a Pong triplet diff --git a/CMakeLists.txt b/CMakeLists.txt index 262c7785..bfe5c58d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,7 +104,7 @@ set(ENV{ROCM_PATH} "${ROCM_PATH}") #================================================================================================== set(TRANSFERBENCH_VERSION_MAJOR 1) set(TRANSFERBENCH_VERSION_MINOR 71) -set(TRANSFERBENCH_VERSION_PATCH_FALLBACK "00") +set(TRANSFERBENCH_VERSION_PATCH_FALLBACK "01") # Auto-compute patch from git: count commits since the last v..* tag. # Falls back to TRANSFERBENCH_VERSION_PATCH_FALLBACK when git is unavailable, diff --git a/docs/reference/presets.rst b/docs/reference/presets.rst index 285371d8..4366a390 100644 --- a/docs/reference/presets.rst +++ b/docs/reference/presets.rst @@ -124,7 +124,7 @@ To modify the behavior of a2a preset, use the following environment variables: - To use only directly connected XGMI links (hop count = 1). 0 = full all-to-all. This can be useful on older MI2XX hardware that doesn't feature full all-to-all XGMI connectivity. Running the standard all-to-all between all pairs of GPUs on that hardware ends up using XGMI links more than once. - - ``1`` + - ``1`` (``0`` on MI4XX, to let DPX route transfers) * - ``A2A_LOCAL`` - To include local transfers (i→i). 0 = exclude, 1 = include. @@ -323,7 +323,7 @@ To modify the behavior of a2asweep preset, use the following environment variabl * - ``A2A_DIRECT`` - To use only directly-connected GPU pairs, set to ``1``. For full all-to-all, set to ``0``. - - ``1`` + - ``1`` (``0`` on MI4XX, to let DPX route transfers) * - ``A2A_LOCAL`` - To include local transfers, set to ``1``. To exclude, set to ``0``. diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index 490ec761..122f4dd0 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -44,7 +44,7 @@ THE SOFTWARE. #include #include -#define CLIENT_VERSION "00" +#define CLIENT_VERSION "01" #include "TransferBench.hpp" using namespace TransferBench; @@ -138,28 +138,34 @@ class EnvVars // Developer features int gpuMaxHwQueues; // Tracks GPU_MAX_HW_QUEUES environment variable + // Detected hardware + std::string gpuArchName; // GCN arch name of GPU 0 (e.g. "gfx942"), empty if no GPU detected + int defaultA2aDirect; // Per-arch default for A2A_DIRECT (0 on MI4XX so DPX can route transfers) + // Constructor that collects values EnvVars() { // Try to detect the GPU hipDeviceProp_t prop; std::string fullName = ""; - std::string archName = ""; int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX); if (numDetectedGpus > 0) { HIP_CALL(hipGetDeviceProperties(&prop, 0)); fullName = prop.gcnArchName; - archName = fullName.substr(0, fullName.find(':')); + gpuArchName = fullName.substr(0, fullName.find(':')); } // Different hardware pick different GPU kernels // This performance difference is generally only noticable when executing fewer CUs int defaultGfxUnroll = 4; - if (archName == "gfx906") defaultGfxUnroll = 8; - else if (archName == "gfx90a") defaultGfxUnroll = 8; - else if (archName == "gfx942") defaultGfxUnroll = 4; - else if (archName == "gfx950") defaultGfxUnroll = 4; - else if (archName == "gfx1250") defaultGfxUnroll = 32; + if (gpuArchName == "gfx906") defaultGfxUnroll = 8; + else if (gpuArchName == "gfx90a") defaultGfxUnroll = 8; + else if (gpuArchName == "gfx942") defaultGfxUnroll = 4; + else if (gpuArchName == "gfx950") defaultGfxUnroll = 4; + else if (gpuArchName == "gfx1250") defaultGfxUnroll = 32; + + // On MI4XX (gfx1250), default to full all-to-all so DPX can route transfers more intuitively + defaultA2aDirect = (gpuArchName == "gfx1250") ? 0 : 1; alwaysValidate = GetEnvVar("ALWAYS_VALIDATE", 0); blockBytes = GetEnvVar("BLOCK_BYTES" , 256); diff --git a/src/client/Presets/AllToAll.hpp b/src/client/Presets/AllToAll.hpp index 3db0709c..f0abc975 100644 --- a/src/client/Presets/AllToAll.hpp +++ b/src/client/Presets/AllToAll.hpp @@ -46,7 +46,7 @@ int AllToAllPreset(EnvVars& ev, int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX); // Collect env vars for this preset - int a2aDirect = EnvVars::GetEnvVar("A2A_DIRECT" , 1); + int a2aDirect = EnvVars::GetEnvVar("A2A_DIRECT" , ev.defaultA2aDirect); int a2aLocal = EnvVars::GetEnvVar("A2A_LOCAL" , 0); int memTypeIdx = EnvVars::GetEnvVar("MEM_TYPE" , 2); int numGpus = EnvVars::GetEnvVar("NUM_GPU_DEVICES", numDetectedGpus); diff --git a/src/client/Presets/AllToAllSweep.hpp b/src/client/Presets/AllToAllSweep.hpp index 36e571d9..58d71118 100644 --- a/src/client/Presets/AllToAllSweep.hpp +++ b/src/client/Presets/AllToAllSweep.hpp @@ -49,7 +49,7 @@ int AllToAllSweepPreset(EnvVars& ev, int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX); // Collect env vars for this preset - int a2aDirect = EnvVars::GetEnvVar("A2A_DIRECT" , 1); + int a2aDirect = EnvVars::GetEnvVar("A2A_DIRECT" , ev.defaultA2aDirect); int a2aLocal = EnvVars::GetEnvVar("A2A_LOCAL" , 0); int memTypeIdx = EnvVars::GetEnvVar("MEM_TYPE" , 2); int numGpus = EnvVars::GetEnvVar("NUM_GPU_DEVICES", numDetectedGpus); diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index f4cc104b..cb991588 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -58,6 +58,7 @@ THE SOFTWARE. #include #include #include +#include #include #include #include @@ -3659,6 +3660,107 @@ const auto& AmdSmiFabricInfoV1(const T& info) return ERR_NONE; } + // Plain-old-data summary of the per-device probe below, sized so it can be + // shipped across a pipe to/from a forked child with a single read/write. + struct IbvProbeResult + { + bool hasActivePort = false; + bool isRoce = false; + int gidIndex = -1; + uint32_t maxMsgSize = 0; + char gidDescriptor[64] = {}; + }; + + // Opens/queries a single IBV device and fills in `result`. Runs inside the + // forked child spawned by ProbeIbvDeviceSafe() below. + static void RunIbvDeviceProbe(ibv_device* devicePtr, IbvProbeResult& result) + { + struct ibv_context *context = ibv_open_device(devicePtr); + if (context) { + struct ibv_device_attr deviceAttr; + if (!ibv_query_device(context, &deviceAttr)) { + int activePort = 0; + for (int port = 1; port <= deviceAttr.phys_port_cnt; ++port) { + struct ibv_port_attr portAttr; + if (ibv_query_port(context, port, &portAttr)) continue; + if (portAttr.state != IBV_PORT_ACTIVE) continue; + + // Any active port may be selected for QP setup via cfg.nic.ibPort, which is + // not visible here, so keep the most restrictive limit across all of them. + if (portAttr.max_msg_sz && + (!result.maxMsgSize || portAttr.max_msg_sz < result.maxMsgSize)) + result.maxMsgSize = portAttr.max_msg_sz; + + // The first active port supplies the RoCE / GID information + if (result.hasActivePort) continue; + activePort = port; + result.hasActivePort = true; + if (portAttr.link_layer == IBV_LINK_LAYER_ETHERNET) { + result.isRoce = true; + std::pair gidInfo (-1, ""); + auto res = GetGidIndex(context, portAttr.gid_tbl_len, activePort, gidInfo); + if (res.errType == ERR_NONE) { + result.gidIndex = gidInfo.first; + strncpy(result.gidDescriptor, gidInfo.second.c_str(), sizeof(result.gidDescriptor) - 1); + } + } + } + } + ibv_close_device(context); + } + } + + // Probes a single IBV device (ibv_open_device/query_device/query_port) in a + // forked child process. Third-party ibverbs provider plugins (e.g. vendor + // RoCE drivers) have been observed to crash inside ibv_open_device() for + // devices that are listed but whose kernel/userspace driver pairing is + // broken; isolating the probe in a child means that crash only costs us + // this one device instead of taking down the whole benchmark. + static bool ProbeIbvDeviceSafe(ibv_device* devicePtr, IbvProbeResult& result) + { + result = IbvProbeResult{}; + + int fds[2]; + if (pipe(fds) != 0) return false; + + pid_t pid = fork(); + if (pid < 0) { + close(fds[0]); + close(fds[1]); + return false; + } + + if (pid == 0) { + // Child: probe the device and report back over the pipe, then exit + // without running parent-side atexit/destructor state. + close(fds[0]); + IbvProbeResult childResult{}; + RunIbvDeviceProbe(devicePtr, childResult); + ssize_t written = write(fds[1], &childResult, sizeof(childResult)); + (void)written; + close(fds[1]); + _exit(0); + } + + // Parent + close(fds[1]); + size_t total = 0; + char* buf = reinterpret_cast(&result); + while (total < sizeof(result)) { + ssize_t n = read(fds[0], buf + total, sizeof(result) - total); + if (n <= 0) break; + total += (size_t)n; + } + close(fds[0]); + + int status = 0; + waitpid(pid, &status, 0); + + bool ok = (total == sizeof(result)) && WIFEXITED(status) && WEXITSTATUS(status) == 0; + if (!ok) result = IbvProbeResult{}; + return ok; + } + static vector& GetIbvDeviceList() { static bool isInitialized = false; @@ -3690,39 +3792,17 @@ const auto& AmdSmiFabricInfoV1(const T& info) ibvDevice.isRoce = false; ibvDevice.gidIndex = -1; { - struct ibv_context *context = ibv_open_device(ibvDevice.devicePtr); - if (context) { - struct ibv_device_attr deviceAttr; - if (!ibv_query_device(context, &deviceAttr)) { - int activePort; - for (int port = 1; port <= deviceAttr.phys_port_cnt; ++port) { - struct ibv_port_attr portAttr; - if (ibv_query_port(context, port, &portAttr)) continue; - if (portAttr.state != IBV_PORT_ACTIVE) continue; - - // Any active port may be selected for QP setup via cfg.nic.ibPort, which is - // not visible here, so keep the most restrictive limit across all of them. - if (portAttr.max_msg_sz && - (!ibvDevice.maxMsgSize || portAttr.max_msg_sz < ibvDevice.maxMsgSize)) - ibvDevice.maxMsgSize = portAttr.max_msg_sz; - - // The first active port supplies the RoCE / GID information - if (ibvDevice.hasActivePort) continue; - activePort = port; - ibvDevice.hasActivePort = true; - if (portAttr.link_layer == IBV_LINK_LAYER_ETHERNET) { - ibvDevice.isRoce = true; - std::pair gidInfo (-1, ""); - auto res = GetGidIndex(context, portAttr.gid_tbl_len, activePort, gidInfo); - if (res.errType == ERR_NONE) { - ibvDevice.gidIndex = gidInfo.first; - ibvDevice.gidDescriptor = gidInfo.second; - } - } - } - } - ibv_close_device(context); + IbvProbeResult probeResult; + if (!ProbeIbvDeviceSafe(ibvDevice.devicePtr, probeResult)) { + fprintf(stderr, "[TransferBench] Warning: probing NIC device '%s' crashed or failed; " + "treating it as inactive. Use TB_NIC_FILTER to exclude it from future runs.\n", + ibvDevice.name.c_str()); } + ibvDevice.hasActivePort = probeResult.hasActivePort; + ibvDevice.isRoce = probeResult.isRoce; + ibvDevice.gidIndex = probeResult.gidIndex; + ibvDevice.maxMsgSize = probeResult.maxMsgSize; + ibvDevice.gidDescriptor = probeResult.gidDescriptor; } ibvDevice.busId = ""; {