diff --git a/cmake/MFCTargets.cmake b/cmake/MFCTargets.cmake index a880b43d3a..13777ccc32 100644 --- a/cmake/MFCTargets.cmake +++ b/cmake/MFCTargets.cmake @@ -200,6 +200,11 @@ exit 0 target_compile_options(${a_target} PRIVATE -fopenmp) target_link_options(${a_target} PRIVATE -fopenmp) elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang") + # GPU architecture: gfx90a (MI250X) unless MFC_AMD_GPU_ARCH overrides it. + set(_amd_gpu_arch "gfx90a") + if (DEFINED ENV{MFC_AMD_GPU_ARCH}) + set(_amd_gpu_arch "$ENV{MFC_AMD_GPU_ARCH}") + endif() # assume-no-nested-parallelism is for correctness, not speed: the two # oversubscription flags are the trigger for llvm/llvm-project#198621 # (a suffix of loop iterations silently skipped), which -O3 happens to @@ -209,7 +214,7 @@ exit 0 # the per-kernel thread-state bookkeeping is dropped (~1.2x on AFAR 24.3). target_compile_options(${a_target} PRIVATE -fopenmp - --offload-arch=gfx90a + --offload-arch=${_amd_gpu_arch} -O3 -fopenmp-assume-threads-oversubscription -fopenmp-assume-teams-oversubscription @@ -227,7 +232,7 @@ exit 0 # longer device link. See docs/documentation/gpuParallelization.md # ("AMD flang known issues") for the failure signature. Still needed on AFAR 24.3 # (cap unchanged; ROCm/llvm-project#4070). - target_link_options(${a_target} PRIVATE -fopenmp --offload-arch=gfx90a -flto-partitions=${MFC_BUILD_JOBS} + target_link_options(${a_target} PRIVATE -fopenmp --offload-arch=${_amd_gpu_arch} -flto-partitions=${MFC_BUILD_JOBS} "SHELL:-Xoffload-linker -mllvm -Xoffload-linker -attributor-max-pi-accesses=16384") # Weak device stubs for symbols AFAR 24.3's device flang runtime @@ -238,7 +243,7 @@ exit 0 set(_rt_stubs_src "${CMAKE_SOURCE_DIR}/cmake/amdflang_device_stubs.c") set(_rt_stubs "${CMAKE_CURRENT_BINARY_DIR}/${a_target}_amdflang_device_stubs.o") add_custom_command(OUTPUT "${_rt_stubs}" - COMMAND "${MFC_AMDCLANG}" -O2 -fopenmp --offload-arch=gfx90a -c "${_rt_stubs_src}" -o "${_rt_stubs}" + COMMAND "${MFC_AMDCLANG}" -O2 -fopenmp --offload-arch=${_amd_gpu_arch} -c "${_rt_stubs_src}" -o "${_rt_stubs}" DEPENDS "${_rt_stubs_src}") target_sources(${a_target} PRIVATE "${_rt_stubs}") endif() diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh index a62f41238f..87d1fecb86 100644 --- a/toolchain/bootstrap/modules.sh +++ b/toolchain/bootstrap/modules.sh @@ -40,7 +40,7 @@ done if [ -v $u_c ]; then log "Select a system:" log "$G""ORNL$W: Ascent (a) | Frontier (f) | Frontier_amd (famd) | Wombat (w)" - log "$B""LLNL $W: Tuolumne (tuo)" + log "$B""LLNL $W: Tuolumne (tuo) | Tuolumne_amd (tamd)" log "$C""ACCESS$W: Bridges2 (b) | Expanse (e) | Delta (d) | DeltaAI (dai)" log "$Y""Gatech$W: Phoenix (p) | Phoenix IFX (pifx)" log "$C""CSCS$W: Santis (san)" @@ -52,7 +52,7 @@ if [ -v $u_c ]; then log "$OR""Florida$W: HiPerGator (h)" log "$C""WPI $W: Turing (t)" log "$R""AMD$W: HPCFund (amdfund)" - log_n "(${G}a${W}/${G}f${W}/${G}famd${W}/${G}w${W}/${B}tuo${W}/${C}b${W}/${C}e${CR}/${C}d/${C}dai${CR}/${Y}p${CR}/${Y}pifx${CR}/${C}san${CR}/${R}r${CR}/${B}cc${CR}/${B}c${CR}/${B}n${CR}/${BR}o${CR}/${BR}pa${CR}/${OR}i${CR}/${OR}h${CR}/${C}t${CR}/${R}amdfund${CR}): " + log_n "(${G}a${W}/${G}f${W}/${G}famd${W}/${G}w${W}/${B}tuo${W}/${B}tamd${W}/${C}b${W}/${C}e${CR}/${C}d/${C}dai${CR}/${Y}p${CR}/${Y}pifx${CR}/${C}san${CR}/${R}r${CR}/${B}cc${CR}/${B}c${CR}/${B}n${CR}/${BR}o${CR}/${BR}pa${CR}/${OR}i${CR}/${OR}h${CR}/${C}t${CR}/${R}amdfund${CR}): " read u_c log fi @@ -228,13 +228,18 @@ if [ -n "$UNLOAD_MODULES" ]; then fi # Don't check for Cray paths on Carpenter, otherwise do check if they exist -if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ] && [ "$u_c" '!=' 'famd' ] ; then +if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ] && [ "$u_c" '!=' 'famd' ] && [ "$u_c" '!=' 'tamd' ] ; then ok "Found $M\$CRAY_LD_LIBRARY_PATH$CR. Prepending to $M\$LD_LIBRARY_PATH$CR." export LD_LIBRARY_PATH="$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH" fi -if [ "$u_c" '==' 'famd' ]; then - export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/sw/crusher/ums/compilers/afar/therock-afar-24.3.0-multiarch-10.1.0-592954c}" +# Frontier (famd) and Tuolumne (tamd): AFAR LLVM Flang (amdflang) + Cray MPICH. +if [ "$u_c" '==' 'famd' ] || [ "$u_c" '==' 'tamd' ]; then + if [ "$u_c" '==' 'famd' ]; then + export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/sw/crusher/ums/compilers/afar/therock-afar-24.3.0-multiarch-10.1.0-592954c}" + else + export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/p/lustre5/bryngelson1/afar/therock-afar-24.3.0-multiarch-10.1.0-592954c}" + fi export PATH=${OLCF_AFAR_ROOT}/lib/llvm/bin:${PATH} export LD_LIBRARY_PATH=${OLCF_AFAR_ROOT}/lib:${OLCF_AFAR_ROOT}/lib/llvm/lib:${LD_LIBRARY_PATH} diff --git a/toolchain/modules b/toolchain/modules index fa399a5016..5557661e92 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -73,6 +73,14 @@ tuo-all cray-fftw/3.3.10.9 cray-hdf5 python/3.12.2 cmake tuo-gpu craype-accel-amd-gfx942 tuo-gpu HSA_XNACK=0 +tamd LLNL Tuolumne AMD +tamd-all python cmake +tamd-all PrgEnv-amd +tamd-all cpe/26.03 cray-pmi +tamd-all MFC_AMD_GPU_ARCH=gfx942 +# amdflang picks the newest gcc-toolset (15), which lacks libquadmath.so; use 14's. +tamd-all LIBRARY_PATH=/opt/rh/gcc-toolset-14/root/usr/lib/gcc/x86_64-redhat-linux/14:$LIBRARY_PATH + d NCSA Delta d-all python d-cpu gcc/11.4.0 openmpi diff --git a/toolchain/templates/tamd.mako b/toolchain/templates/tamd.mako new file mode 100644 index 0000000000..7965b53028 --- /dev/null +++ b/toolchain/templates/tamd.mako @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +<%namespace name="helpers" file="helpers.mako"/> + +% if engine == 'batch': +# flux: -N ${nodes} +# flux: -n ${tasks_per_node*nodes} +# flux: --job-name="${name}" +# flux: --output="${name}.out" +# flux: --error="${name}.err" +# flux: --time-limit=${walltime} +# flux: --exclusive +# flux: --setattr=thp=always +% if account: +# flux: --bank=${account} +% endif +% if partition: +# flux: --queue=${partition} +% endif +% endif + +${helpers.template_prologue()} + +ok ":) Loading modules:\n" +cd "${MFC_ROOT_DIR}" +% if engine == 'batch': +. ./mfc.sh load -c tamd -m ${'g' if gpu_enabled else 'c'} +% endif +cd - > /dev/null +echo + +export HSA_XNACK=0 + +% for target in targets: + ${helpers.run_prologue(target)} + + % if not mpi: + (set -x; ${profiler} "${target.get_install_binpath(case)}") + % else: + (set -x; flux run \ + --nodes=${nodes} --ntasks=${tasks_per_node * nodes} \ + -o spindle.level=off --exclusive \ + % if gpu_enabled: + --gpus-per-task 1 \ + % endif + ${profiler} "${target.get_install_binpath(case)}") + % endif + + ${helpers.run_epilogue(target)} + + echo +% endfor + +${helpers.template_epilogue()}