Add Tuolumne AMD (tamd): AFAR 24.3.0 amdflang + Cray MPICH, gfx942

cmake/MFCTargets.cmake

@@ -200,6 +200,11 @@ exit 0
                     target_compile_options(${a_target} PRIVATE -fopenmp)
                     target_link_options(${a_target} PRIVATE -fopenmp)
                 elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+                    # GPU architecture: gfx90a (MI250X) unless MFC_AMD_GPU_ARCH overrides it.
+                    set(_amd_gpu_arch "gfx90a")
+                    if (DEFINED ENV{MFC_AMD_GPU_ARCH})
+                        set(_amd_gpu_arch "$ENV{MFC_AMD_GPU_ARCH}")
+                    endif()
                     # assume-no-nested-parallelism is for correctness, not speed: the two
                     # oversubscription flags are the trigger for llvm/llvm-project#198621
                     # (a suffix of loop iterations silently skipped), which -O3 happens to
@@ -209,7 +214,7 @@ exit 0
                     # the per-kernel thread-state bookkeeping is dropped (~1.2x on AFAR 24.3).
                     target_compile_options(${a_target} PRIVATE
                         -fopenmp
-                        --offload-arch=gfx90a
+                        --offload-arch=${_amd_gpu_arch}
                         -O3
                         -fopenmp-assume-threads-oversubscription
                         -fopenmp-assume-teams-oversubscription
@@ -227,7 +232,7 @@ exit 0
                     # longer device link. See docs/documentation/gpuParallelization.md
                     # ("AMD flang known issues") for the failure signature. Still needed on AFAR 24.3
                     # (cap unchanged; ROCm/llvm-project#4070).
-                    target_link_options(${a_target} PRIVATE -fopenmp --offload-arch=gfx90a -flto-partitions=${MFC_BUILD_JOBS}
+                    target_link_options(${a_target} PRIVATE -fopenmp --offload-arch=${_amd_gpu_arch} -flto-partitions=${MFC_BUILD_JOBS}
                         "SHELL:-Xoffload-linker -mllvm -Xoffload-linker -attributor-max-pi-accesses=16384")
 
                     # Weak device stubs for symbols AFAR 24.3's device flang runtime
@@ -238,7 +243,7 @@ exit 0
                     set(_rt_stubs_src "${CMAKE_SOURCE_DIR}/cmake/amdflang_device_stubs.c")
                     set(_rt_stubs "${CMAKE_CURRENT_BINARY_DIR}/${a_target}_amdflang_device_stubs.o")
                     add_custom_command(OUTPUT "${_rt_stubs}"
-                        COMMAND "${MFC_AMDCLANG}" -O2 -fopenmp --offload-arch=gfx90a -c "${_rt_stubs_src}" -o "${_rt_stubs}"
+                        COMMAND "${MFC_AMDCLANG}" -O2 -fopenmp --offload-arch=${_amd_gpu_arch} -c "${_rt_stubs_src}" -o "${_rt_stubs}"
                         DEPENDS "${_rt_stubs_src}")
                     target_sources(${a_target} PRIVATE "${_rt_stubs}")
                 endif()

toolchain/bootstrap/modules.sh

@@ -40,7 +40,7 @@ done
 if [ -v $u_c ]; then
     log   "Select a system:"
     log   "$G""ORNL$W:    Ascent     (a) | Frontier (f) | Frontier_amd (famd) | Wombat (w)"
-    log   "$B""LLNL $W:   Tuolumne   (tuo)"
+    log   "$B""LLNL $W:   Tuolumne   (tuo) | Tuolumne_amd (tamd)"
     log   "$C""ACCESS$W:  Bridges2   (b) | Expanse (e) | Delta  (d) | DeltaAI (dai)"
     log   "$Y""Gatech$W:  Phoenix    (p) | Phoenix IFX (pifx)"
     log   "$C""CSCS$W:    Santis     (san)"
@@ -52,7 +52,7 @@ if [ -v $u_c ]; then
     log   "$OR""Florida$W: HiPerGator (h)"
     log   "$C""WPI $W:   Turing   (t)"
     log   "$R""AMD$W:     HPCFund  (amdfund)"
-    log_n "(${G}a${W}/${G}f${W}/${G}famd${W}/${G}w${W}/${B}tuo${W}/${C}b${W}/${C}e${CR}/${C}d/${C}dai${CR}/${Y}p${CR}/${Y}pifx${CR}/${C}san${CR}/${R}r${CR}/${B}cc${CR}/${B}c${CR}/${B}n${CR}/${BR}o${CR}/${BR}pa${CR}/${OR}i${CR}/${OR}h${CR}/${C}t${CR}/${R}amdfund${CR}): "
+    log_n "(${G}a${W}/${G}f${W}/${G}famd${W}/${G}w${W}/${B}tuo${W}/${B}tamd${W}/${C}b${W}/${C}e${CR}/${C}d/${C}dai${CR}/${Y}p${CR}/${Y}pifx${CR}/${C}san${CR}/${R}r${CR}/${B}cc${CR}/${B}c${CR}/${B}n${CR}/${BR}o${CR}/${BR}pa${CR}/${OR}i${CR}/${OR}h${CR}/${C}t${CR}/${R}amdfund${CR}): "
     read u_c
     log
 fi
@@ -228,13 +228,18 @@ if [ -n "$UNLOAD_MODULES" ]; then
 fi
 
 # Don't check for Cray paths on Carpenter, otherwise do check if they exist
-if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ] &&  [ "$u_c" '!=' 'famd' ] ; then
+if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ] &&  [ "$u_c" '!=' 'famd' ] && [ "$u_c" '!=' 'tamd' ] ; then
     ok "Found $M\$CRAY_LD_LIBRARY_PATH$CR. Prepending to $M\$LD_LIBRARY_PATH$CR."
     export LD_LIBRARY_PATH="$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH"
 fi
 
-if [ "$u_c" '==' 'famd' ]; then 
-    export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/sw/crusher/ums/compilers/afar/therock-afar-24.3.0-multiarch-10.1.0-592954c}"
+# Frontier (famd) and Tuolumne (tamd): AFAR LLVM Flang (amdflang) + Cray MPICH.
+if [ "$u_c" '==' 'famd' ] || [ "$u_c" '==' 'tamd' ]; then
+    if [ "$u_c" '==' 'famd' ]; then
+        export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/sw/crusher/ums/compilers/afar/therock-afar-24.3.0-multiarch-10.1.0-592954c}"
+    else
+        export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/p/lustre5/bryngelson1/afar/therock-afar-24.3.0-multiarch-10.1.0-592954c}"
+    fi
 
     export PATH=${OLCF_AFAR_ROOT}/lib/llvm/bin:${PATH}
     export LD_LIBRARY_PATH=${OLCF_AFAR_ROOT}/lib:${OLCF_AFAR_ROOT}/lib/llvm/lib:${LD_LIBRARY_PATH}

toolchain/modules

@@ -73,6 +73,14 @@ tuo-all cray-fftw/3.3.10.9 cray-hdf5 python/3.12.2 cmake
 tuo-gpu craype-accel-amd-gfx942
 tuo-gpu HSA_XNACK=0
 
+tamd    LLNL Tuolumne AMD
+tamd-all python cmake
+tamd-all PrgEnv-amd
+tamd-all cpe/26.03 cray-pmi
+tamd-all MFC_AMD_GPU_ARCH=gfx942
+# amdflang picks the newest gcc-toolset (15), which lacks libquadmath.so; use 14's.
+tamd-all LIBRARY_PATH=/opt/rh/gcc-toolset-14/root/usr/lib/gcc/x86_64-redhat-linux/14:$LIBRARY_PATH
+
 d     NCSA Delta
 d-all python
 d-cpu gcc/11.4.0 openmpi

toolchain/templates/tamd.mako

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+<%namespace name="helpers" file="helpers.mako"/>
+
+% if engine == 'batch':
+# flux: -N ${nodes}
+# flux: -n ${tasks_per_node*nodes}
+# flux: --job-name="${name}"
+# flux: --output="${name}.out"
+# flux: --error="${name}.err"
+# flux: --time-limit=${walltime}
+# flux: --exclusive
+# flux: --setattr=thp=always
+% if account:
+# flux: --bank=${account}
+% endif
+% if partition:
+# flux: --queue=${partition}
+% endif
+% endif
+
+${helpers.template_prologue()}
+
+ok ":) Loading modules:\n"
+cd "${MFC_ROOT_DIR}"
+% if engine == 'batch':
+. ./mfc.sh load -c tamd -m ${'g' if gpu_enabled else 'c'}
+% endif
+cd - > /dev/null
+echo
+
+export HSA_XNACK=0
+
+% for target in targets:
+    ${helpers.run_prologue(target)}
+
+    % if not mpi:
+        (set -x; ${profiler} "${target.get_install_binpath(case)}")
+    % else:
+        (set -x; flux run \
+            --nodes=${nodes} --ntasks=${tasks_per_node * nodes} \
+            -o spindle.level=off --exclusive \
+            % if gpu_enabled:
+                --gpus-per-task 1 \
+            % endif
+            ${profiler} "${target.get_install_binpath(case)}")
+    % endif
+
+    ${helpers.run_epilogue(target)}
+
+    echo
+% endfor
+
+${helpers.template_epilogue()}