[SYCL] Support for SYCL port in parity with CUDA

.github/workflows/sycl-cpu-ci.yml

@@ -0,0 +1,79 @@
+name: Build and Test - SYCL CPU
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize, labeled]
+
+  workflow_dispatch:
+
+env:
+  GH_ACTIONS_TOOLCHAIN: .github/workflows/toolchains/gh-actions.cmake
+  ENV_PREFIX_PATH: "/opt/intel/oneapi/mkl/latest"
+
+jobs:
+  sycl_cpu_build:
+    name: SYCL CPU Build and Test
+    if: >
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'SYCL') ||
+      (github.event_name == 'pull_request' && github.event.action != 'labeled' && contains(github.event.pull_request.title, '[SYCL]'))
+    runs-on: ubuntu-latest
+    container:
+        image: intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04
+    timeout-minutes: 60
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install HDF5 development headers
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install -y libhdf5-dev
+
+    - name: Setup Compiler
+      shell: bash
+      run: |
+        echo "set(CMAKE_C_COMPILER   icx   CACHE STRING \"\" FORCE)" >> ${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+        echo "set(CMAKE_CXX_COMPILER icpx  CACHE STRING \"\" FORCE)" >> ${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+
+    - name: Setup Build Type
+      shell: bash
+      run: echo "set(CMAKE_BUILD_TYPE Release CACHE BOOL \"\" FORCE)" >>
+             ${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+
+    - name: Setup Enable SYCL
+      shell: bash
+      run: |
+        echo "set(GAUXC_ENABLE_SYCL  ON              CACHE BOOL   \"\" FORCE)" >> ${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+        echo "set(GAUXC_SYCL_TARGET  spir64_x86_64   CACHE STRING \"\" FORCE)" >> ${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+        echo "set(GAUXC_ENABLE_MPI   OFF             CACHE BOOL   \"\" FORCE)" >> ${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+
+    - name: Configure CMake
+      shell: bash
+      run: cmake -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build
+                 -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/install
+                 -DCMAKE_PREFIX_PATH=${ENV_PREFIX_PATH}
+                 -DCMAKE_TOOLCHAIN_FILE=${GITHUB_WORKSPACE}/${GH_ACTIONS_TOOLCHAIN}
+
+    - name: Build
+      shell: bash
+      timeout-minutes: 35
+      run: cmake --build ${{runner.workspace}}/build -j2
+
+    - name: List SYCL devices
+      shell: bash
+      run: sycl-ls || true
+
+    # The SYCL kernels run on the OpenCL CPU device here, which is orders of
+    # magnitude slower than a GPU: the XC integrator sections in particular are
+    # minutes apiece on a many-core host and far worse on a 2-core runner. Cap
+    # the step so a hang is reported against the test rather than burning the
+    # job's whole budget, and give ctest its own per-test timeout so a single
+    # wedged case cannot stall the rest.
+    - name: Test
+      shell: bash
+      timeout-minutes: 20
+      env:
+        ONEAPI_DEVICE_SELECTOR: opencl:cpu
+      run: ctest --test-dir ${{runner.workspace}}/build --output-on-failure --timeout 900

CMakeLists.txt

@@ -31,6 +31,7 @@ option( GAUXC_ENABLE_C          "Enable C API"                ON  )
 option( GAUXC_ENABLE_HOST       "Enable Host Integrator"      ON  )
 option( GAUXC_ENABLE_CUDA       "Enable CUDA Bindings"        OFF )
 option( GAUXC_ENABLE_HIP        "Enable HIP Bindings"         OFF )
+option( GAUXC_ENABLE_SYCL       "Enable SYCL Bindings"        OFF )
 option( GAUXC_ENABLE_MPI        "Enable MPI Bindings"         ON  )
 option( GAUXC_ENABLE_OPENMP     "Enable OpenMP Compilation"   ON  )
 option( GAUXC_ENABLE_TESTS      "Enable Unit Tests"           ON  )
@@ -41,10 +42,25 @@ option( GAUXC_BLAS_PREFER_ILP64 "Prefer ILP64 for host BLAS"  OFF )
 option( GAUXC_LINK_CUDA_STATIC  "Link GauXC with static CUDA libs"  OFF )
 
 include(CMakeDependentOption)
+
+# MAGMA has no released SYCL backend: upstream master carries only the CUDA and
+# HIP backends, and the SYCL work lives on an unmerged branch. Rather than
+# silently dropping the request, fail loudly if the user asked for both.
+if( GAUXC_ENABLE_SYCL AND GAUXC_ENABLE_MAGMA )
+  message( FATAL_ERROR
+    "GAUXC_ENABLE_MAGMA=ON is not supported with GAUXC_ENABLE_SYCL=ON: MAGMA "
+    "provides no SYCL backend. Batched linear algebra for the SYCL backend is "
+    "served by oneMKL instead; configure with -DGAUXC_ENABLE_MAGMA=OFF." )
+endif()
+
 cmake_dependent_option( GAUXC_ENABLE_MAGMA    
   "Enable MAGMA Linear Algebra"           ON
   "GAUXC_ENABLE_CUDA OR GAUXC_ENABLE_HIP" OFF  
 )
+cmake_dependent_option( GAUXC_ENABLE_ONECCL
+  "Enable oneCCL Collectives" OFF
+  "GAUXC_ENABLE_SYCL"         OFF
+)
 cmake_dependent_option( GAUXC_ENABLE_NCCL 
   "Enable NCCL Collectives" OFF  
   "GAUXC_ENABLE_CUDA"       OFF 
@@ -59,12 +75,14 @@ set( GAUXC_HAS_C          FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_HOST       FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_CUDA       FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_HIP        FALSE CACHE BOOL "" FORCE )
+set( GAUXC_HAS_SYCL       FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_MPI        FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_OPENMP     FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_GAU2GRID   FALSE CACHE STRING "" FORCE )
 set( GAUXC_HAS_HDF5       FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_MAGMA      FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_NCCL       FALSE CACHE BOOL "" FORCE )
+set( GAUXC_HAS_ONECCL     FALSE CACHE BOOL "" FORCE )
 set( GAUXC_HAS_CUTLASS    FALSE CACHE BOOL "" FORCE )
 set( GAUXC_BLAS_IS_LP64   FALSE CACHE BOOL "" FORCE )
 
@@ -73,12 +91,14 @@ mark_as_advanced( FORCE
   GAUXC_HAS_HOST     
   GAUXC_HAS_CUDA     
   GAUXC_HAS_HIP      
+  GAUXC_HAS_SYCL     
   GAUXC_HAS_MPI      
   GAUXC_HAS_OPENMP   
   GAUXC_HAS_GAU2GRID 
   GAUXC_HAS_HDF5     
   GAUXC_HAS_MAGMA    
   GAUXC_HAS_NCCL     
+  GAUXC_HAS_ONECCL   
   GAUXC_HAS_CUTLASS  
   GAUXC_BLAS_IS_LP64 
 )
@@ -104,8 +124,44 @@ if( GAUXC_ENABLE_HIP )
   set( GAUXC_HAS_HIP TRUE CACHE BOOL "GauXC has HIP and will build HIP bindings" FORCE )
 endif()
 
+if( GAUXC_ENABLE_SYCL )
+  # SYCL is compiled through the CXX compiler; no separate language to enable
+  set( GAUXC_HAS_SYCL TRUE CACHE BOOL "GauXC has SYCL and will build SYCL bindings" FORCE )
+
+  # e.g. intel_gpu_pvc | spir64_x86_64 | nvidia_gpu_sm_80 | amd_gpu_gfx90a
+  # Declared here (rather than in gauxc-sycl) so that it is visible to the
+  # ExchCXX pass-through, which is configured before the gauxc target exists.
+  #
+  # Defaults to ahead-of-time for PVC: that is how these builds are deployed,
+  # and a JIT build pays the whole device-image compile on the first kernel
+  # launch of every run, which is easily minutes for the collocation and
+  # ExchCXX images. Set to an empty string explicitly for a JIT build.
+  set( GAUXC_SYCL_TARGET "intel_gpu_pvc" CACHE STRING
+       "Alias for -fsycl-targets (see Users Manual); empty for a JIT build" )
+
+  # SYCL device code shares the CXX FP model with the host, and ExchCXX's SYCL
+  # bindings require IEEE semantics for agreement with the reference values
+  include( CheckCXXCompilerFlag )
+  check_cxx_compiler_flag( "-fp-model=precise" GAUXC_CXX_HAS_FP_MODEL_PRECISE )
+  if( GAUXC_CXX_HAS_FP_MODEL_PRECISE )
+    set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fp-model=precise" )
+  endif()
+
+  # Claim SYCL::SYCL before ExchCXX is fetched. ExchCXX ships its own FindSYCL
+  # which attaches -fsycl as an INTERFACE_COMPILE_OPTION; were that definition
+  # to win, every GauXC translation unit would run the SYCL frontend, including
+  # the large host-only sources where it buys nothing. GauXC's own FindSYCL
+  # keeps the target link-only and stamps -fsycl per source instead.
+  find_package( SYCL REQUIRED )
+endif()
+
+# Device backends are mutually exclusive
+if( (GAUXC_HAS_CUDA AND GAUXC_HAS_SYCL) OR (GAUXC_HAS_HIP AND GAUXC_HAS_SYCL) )
+  message( FATAL_ERROR "SYCL bindings are mutually exclusive with CUDA / HIP" )
+endif()
+
 # Decided if we're compiling device bindings
-if( GAUXC_HAS_CUDA OR GAUXC_HAS_HIP )
+if( GAUXC_HAS_CUDA OR GAUXC_HAS_HIP OR GAUXC_HAS_SYCL )
   set( GAUXC_HAS_DEVICE TRUE CACHE BOOL "Enable Device Code" )
 else()
   set( GAUXC_HAS_DEVICE FALSE CACHE BOOL "Enable Device Code" )

README.md

@@ -202,12 +202,15 @@ target_link_libraries( my_target PUBLIC gauxc::gauxc )
 | `GAUXC_ENABLE_HOST`        | Enable HOST integrators                                   | `ON`     |
 | `GAUXC_ENABLE_CUDA`        | Enable CUDA integrators                                   | `OFF`    |
 | `GAUXC_ENABLE_HIP`         | Enable HIP integrators                                    | `OFF`    |
+| `GAUXC_ENABLE_SYCL`        | Enable SYCL integrators                                   | `OFF`    |
 | `GAUXC_ENABLE_MAGMA`       | Enable MAGMA for batched BLAS (No effect if no GPU)       | `ON`     |
 | `GAUXC_ENABLE_CUTLASS`     | Enable CUTLASS for batched BLAS (No effect if no CUDA)    | `OFF`    |
 | `GAUXC_ENABLE_NCCL`        | Enable NCCL bindings for topology aware GPU reductions    | `OFF`    |
+| `GAUXC_ENABLE_ONECCL`      | Enable oneCCL bindings for GPU reductions (SYCL only)     | `OFF`    |
 | `GAUXC_ENABLE_MPI`         | Enable MPI Bindings                                       | `ON`     |
 | `GAUXC_ENABLE_OPENMP`      | Enable OpenMP Bindings                                    | `ON`     |
 | `CMAKE_CUDA_ARCHITECTURES` | CUDA architechtures (e.g. 70 for Volta, 80 for Ampere)    |  --      |
+| `GAUXC_SYCL_TARGET`        | SYCL AoT target (e.g. `intel_gpu_pvc`); JIT if unset      |  --      |
 | `BLAS_LIBRARIES`           | Full BLAS linker.                                         |  --      |
 | `MAGMA_ROOT_DIR`           | Install prefix for MAGMA.                                 |  --      |
 

cmake/gauxc-config.cmake.in

@@ -14,8 +14,10 @@ set( GAUXC_HAS_C          @GAUXC_HAS_C@         )
 set( GAUXC_HAS_HOST       @GAUXC_HAS_HOST@      )
 set( GAUXC_HAS_CUDA       @GAUXC_HAS_CUDA@      )
 set( GAUXC_HAS_HIP        @GAUXC_HAS_HIP@       )
+set( GAUXC_HAS_SYCL       @GAUXC_HAS_SYCL@      )
 set( GAUXC_HAS_MAGMA      @GAUXC_HAS_MAGMA@     )
 set( GAUXC_HAS_NCCL       @GAUXC_HAS_NCCL@      )
+set( GAUXC_HAS_ONECCL     @GAUXC_HAS_ONECCL@    )
 set( GAUXC_HAS_CUTLASS    @GAUXC_HAS_CUTLASS@   )
 set( GAUXC_HAS_MPI        @GAUXC_HAS_MPI@       )
 set( GAUXC_HAS_OPENMP     @GAUXC_HAS_OPENMP@    )
@@ -56,6 +58,25 @@ if( GAUXC_HAS_CUDA )
   endif()
 endif()
 
+if( GAUXC_HAS_SYCL )
+  find_dependency( SYCL )
+  # MKLConfig defaults to ILP64 and keeps separate cache entries for the host
+  # and SYCL domains; both must match the interface GauXC was built against or
+  # the duplicate dgemm_ binds to the wrong copy.
+  if( GAUXC_BLAS_IS_LP64 )
+    set( _gauxc_mkl_interface intel_lp64 )
+  else()
+    set( _gauxc_mkl_interface intel_ilp64 )
+  endif()
+  set( MKL_INTERFACE_FULL      ${_gauxc_mkl_interface} CACHE STRING "" FORCE )
+  set( MKL_SYCL_INTERFACE_FULL ${_gauxc_mkl_interface} CACHE STRING "" FORCE )
+  unset( _gauxc_mkl_interface )
+  find_dependency( MKL CONFIG )
+  if( GAUXC_HAS_ONECCL )
+    find_dependency( oneCCL )
+  endif()
+endif()
+
 if( GAUXC_HAS_MPI )
   find_dependency( MPI )
 endif()

cmake/gauxc-exchcxx.cmake

@@ -9,8 +9,13 @@ if( NOT ${ExchCXX_FOUND} )
 
   set( EXCHCXX_ENABLE_CUDA  ${GAUXC_HAS_CUDA} CACHE BOOL "" )
   set( EXCHCXX_ENABLE_HIP   ${GAUXC_HAS_HIP}  CACHE BOOL "" )
+  set( EXCHCXX_ENABLE_SYCL  ${GAUXC_HAS_SYCL} CACHE BOOL "" )
   set( EXCHCXX_ENABLE_TESTS OFF               CACHE BOOL "" )
 
+  if( GAUXC_HAS_SYCL AND GAUXC_SYCL_TARGET )
+    set( EXCHCXX_SYCL_TARGET ${GAUXC_SYCL_TARGET} CACHE STRING "" )
+  endif()
+
   FetchContent_Declare(
     exchcxx
     GIT_REPOSITORY ${GAUXC_EXCHCXX_REPOSITORY} 
@@ -30,6 +35,10 @@ else()
     message( FATAL_ERROR "GauXC HIP BINDINGS REQUIRE ExchCXX HIP Bindings" )
   endif()
 
+  if( ${GAUXC_HAS_SYCL} AND NOT ${EXCHCXX_ENABLE_SYCL} )
+    message( FATAL_ERROR "GauXC SYCL BINDINGS REQUIRE ExchCXX SYCL Bindings" )
+  endif()
+
 endif()