From b600a96d2b206f43292506a70facc5a306600886 Mon Sep 17 00:00:00 2001 From: DENEL Bertrand Date: Thu, 27 Aug 2026 15:10:14 -0500 Subject: [PATCH 01/22] Clean up --- scripts/spack_configs/maple/spack.yaml | 161 --------------- scripts/spack_configs/pine/spack.yaml | 186 ------------------ .../spack_packages/packages/geosx/package.py | 2 +- 3 files changed, 1 insertion(+), 348 deletions(-) delete mode 100644 scripts/spack_configs/maple/spack.yaml delete mode 100644 scripts/spack_configs/pine/spack.yaml diff --git a/scripts/spack_configs/maple/spack.yaml b/scripts/spack_configs/maple/spack.yaml deleted file mode 100644 index 82594316..00000000 --- a/scripts/spack_configs/maple/spack.yaml +++ /dev/null @@ -1,161 +0,0 @@ -# This is a Spack Environment file for Maple. -# -# It describes a set of packages to be installed, along with -# configuration settings. -# -# Run command from the top-level of the repository: -# ./scripts/uberenv/uberenv.py \ -# --spec "%clang@17.0.4 +cuda~pygeosx~docs cuda_arch=90 ^cuda@12.4.131+allow-unsupported-compilers" \ -# --spack-env-file=scripts/spack_configs/maple/spack.yaml \ -# --project-json=.uberenv_config.json \ -# --prefix ${GEOS_TPL_DIR} -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage: - - $spack/../build_stage - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - ############# - # COMPILERS # - ############# - compilers: - - compiler: - spec: clang@17.0.4 - paths: - cc: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/llvm-17.0.4-24xg7dvliazr7dqlcimhsgpog3nvdiie/bin/clang - cxx: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/llvm-17.0.4-24xg7dvliazr7dqlcimhsgpog3nvdiie/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: - cflags: - cxxflags: - operating_system: rocky9 - target: aarch64 - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] - - ############# - # PACKAGES # - ############# - packages: - all: - target: [neoverse_n1] - compiler: [clang] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] - - #### - # make sure spack doesn't rebuild mpi and mathpresso libs - mpi: - buildable: false - - mathpresso: - buildable: false - - #### - # spec of spack packages to reuse - bison: - externals: - - spec: bison@3.8.2 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/bison-3.8.2-mkim26leceonxvdvlpukc4y65kkaa4j3/ - buildable: false - gmp: - externals: - - spec: gmp@6.2.1 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/gmp-6.2.1-h7dxik46ba5ebtbvpdtfqm7kanbxokzm - buildable: false - mpfr: - externals: - - spec: mpfr@4.2.0 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/mpfr-4.2.0-i6bud7m6k6lmduamd6h5gw62xdf2wytf - buildable: false - libiconv: - externals: - - spec: libiconv@1.17 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/libiconv-1.17-m3y2ti2b4r73c2w2d4tbln6bdlozblq6 - buildable: false - perl: - externals: - - spec: perl@5.38.0 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/perl-5.38.0-vjg3pfiwld5tk2vu22ji2f2fgrgxrekn - buildable: false - berkeley-db: - externals: - - spec: berkeley-db@18.1.40 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/berkeley-db-18.1.40-nqcmpq7kkd5nqo7x6dd7tvcahv7fk5nd - buildable: false - cmake: - externals: - - spec: cmake@3.29.6 - prefix: /hrtc/apps/devtools/spack/MAPLE/linux-rocky9-neoverse_v2/gcc-12.2.0/cmake-3.29.6-yl7bm5nfishcx3d6dlkqfj7pc2muvowe - buildable: false - - #### - # spec of system packages to reuse - # bin - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /usr - buildable: false - git: - externals: - - spec: git@2.39.3~tcltk - prefix: /usr - buildable: false - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - buildable: false - m4: - externals: - - spec: m4@1.4.19 - prefix: /usr - buildable: false - python: - buildable: False - externals: - - spec: python@3.9.18 - prefix: /usr - #lib - cuda: - externals: - - spec: cuda@12.4.131 - prefix: /hrtc/apps/cuda/12.4.131/aarch64/rocky9/ - buildable: false - openmpi: - externals: - - spec: openmpi@5.0.5 - prefix: /hrtc/apps/mpi/openmpi/aarch64/rocky9/5.0.5/llvm/17.0.4/cuda.12.4 - buildable: false - pkgconf: - externals: - - spec: pkgconf@3.0.0 - prefix: /usr - buildable: false - readline: - externals: - - spec: readline@8.1 - prefix: /usr - buildable: false - - diff --git a/scripts/spack_configs/pine/spack.yaml b/scripts/spack_configs/pine/spack.yaml deleted file mode 100644 index 6141aec7..00000000 --- a/scripts/spack_configs/pine/spack.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# This is a Spack Environment file for Pine (Spack 1.0.2) -# Run command from the top-level of the repository: -# ./scripts/uberenv/uberenv.py \ -# --spec "~openmp~pygeosx~docs %gcc-11" \ -# --spack-env-file=scripts/spack_configs/pine/spack.yaml \ -# --project-json=.uberenv_config.json \ -# --prefix ${GEOS_TPL_DIR} - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage: - - $spack/../build_stage - build_jobs: 16 - - view: false - - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - gcc-11: - - spec: '%c=gcc@11.4.1' - when: '%c' - - spec: '%cxx=gcc@11.4.1' - when: '%cxx' - - spec: '%fortran=gcc@11.4.1' - when: '%fortran' - - spec: '%openmpi@5.0.5' - when: '%mpi' - - packages: - all: - target: [zen4] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] - zlib-api: [zlib] - pkgconfig: [pkgconf] - - # Force Spack to build openblas, not use system version - openblas: - buildable: true - require: "@0.3.27 threads=none" - - # Disable unwanted compilers - nvhpc: - buildable: false - intel: - buildable: false - oneapi: - buildable: false - cce: - buildable: false - llvm-amdgpu: - buildable: false - - gcc: - externals: - - spec: gcc@=11.4.1 - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - flags: - cflags: -march=native -mtune=native - cxxflags: -march=native -mtune=native - environment: {} - extra_rpaths: [] - buildable: false - - # libgcc_s.so.1 is missing in /usr/lib/gcc/x86_64-redhat-linux/11 so removing it for now - # gcc-runtime: - # buildable: false - # externals: - # - spec: gcc-runtime@11.4.1 - # prefix: /usr/lib/gcc/x86_64-redhat-linux/11 - - gmp: - externals: - - spec: gmp@6.3.0 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/gmp-6.3.0-q4glkxa77fd4rejkrrsuydut2c222fkm - buildable: false - - mpfr: - externals: - - spec: mpfr@4.2.0 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/mpfr-4.2.1-eaxjflobjpilpwfe4aalusexhcxhptrf - buildable: false - - libiconv: - externals: - - spec: libiconv@1.17 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/libiconv-1.17-crgtppb7hj54qwkl3hfc6ejyu34gsa4d - buildable: false - - perl: - externals: - - spec: perl@5.38.2 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/perl-5.38.2-wxq42supt6zs3f63ajjtp6423q6teyot - buildable: false - - berkeley-db: - externals: - - spec: berkeley-db@18.1.40 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/berkeley-db-18.1.40-nixwddyxlb523z3aly4b7bje4rfft3me - buildable: false - - cmake: - externals: - - spec: cmake@3.29.6 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/cmake-3.29.6-i23jd7kq7lyqvrpsguhznfhoy4mso6up - buildable: false - - git: - externals: - - spec: git@2.39.3~tcltk - prefix: /usr - buildable: false - - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - buildable: false - - m4: - externals: - - spec: m4@1.4.19 - prefix: /usr - buildable: false - - python: - buildable: false - externals: - - spec: python@3.9.18 - prefix: /usr - - bison: - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - - ninja: - buildable: false - externals: - - spec: ninja@1.10.2 - prefix: /usr - - openmpi: - externals: - - spec: openmpi@5.0.5 - prefix: /hrtc/apps/mpi/openmpi/x86_64/rocky9/5.0.5/gcc/11.4.1/nocuda/ - buildable: false - - pkgconf: - externals: - - spec: pkgconf@3.0.0 - prefix: /usr - buildable: false - - readline: - externals: - - spec: readline@8.1 - prefix: /usr - buildable: false - - zlib: - buildable: false - externals: - - spec: zlib@1.2.11 - prefix: /usr - - concretizer: - unify: true - reuse: false diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 2c095bb8..de59ce3e 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -784,7 +784,7 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) + cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) From 6edaeb285952611909a354ac6e98103390077d5b Mon Sep 17 00:00:00 2001 From: DENEL Bertrand Date: Thu, 27 Aug 2026 20:36:11 -0500 Subject: [PATCH 02/22] Fixed hash --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ab7fe24a..9a6ecac6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -851,7 +851,7 @@ list(APPEND build_list metis ) if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "https://github.com/xiaoyeli/superlu_dist/archive/0f6efc3.tar.gz") - set(SUPERLU_URL_HASH "49cd47da22fe72a37011e8dfaf94bd63ad8d61cc8144eac7a571bcbd0a2edf58a") + set(SUPERLU_URL_HASH "49cd47da22fe72a37011e8dfaf94bd63ad8d61cc8144eac7a571bcbd0a2edf58") if ( CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 16.0 ) set(SUPERLU_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS}") From 76b8c89958ff738ed061d34c6a62c31d4c66cc03 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 27 Aug 2026 22:46:15 -0400 Subject: [PATCH 03/22] Superbuild fixes --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a6ecac6..f7b4e7d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -259,8 +259,9 @@ ExternalProject_Add( hdf5 URL_HASH SHA256=${HDF5_URL_HASH} PREFIX ${PROJECT_BINARY_DIR}/hdf5 INSTALL_DIR ${HDF5_DIR} - BUILD_COMMAND make -j ${NUM_PROC} - INSTALL_COMMAND make install + BUILD_COMMAND ${TPL_BUILD_COMMAND} + INSTALL_COMMAND ${TPL_INSTALL_COMMAND} + CMAKE_GENERATOR ${TPL_GENERATOR} CMAKE_ARGS -D CMAKE_C_COMPILER:STRING=${HDF5_C_COMPILER} -D CMAKE_CXX_COMPILER:STRING=${HDF5_CXX_COMPILER} -D CMAKE_C_FLAGS:STRING=${C_FLAGS_NO_WARNINGS} @@ -421,6 +422,7 @@ ExternalProject_Add( raja -D RAJA_ENABLE_NV_TOOLS_EXT:BOOL=${ENABLE_CUDA_NVTOOLSEXT} -D RAJA_ENABLE_ROCTX:BOOL=${ENABLE_ROCTX} -D CMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER} + -D CMAKE_CUDA_HOST_COMPILER:FILEPATH=${CMAKE_CUDA_HOST_COMPILER} -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CUB_DIR:PATH=${CUB_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} @@ -439,6 +441,7 @@ ExternalProject_Add( raja -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} + -D RAJA_ENABLE_EXERCISES:BOOL=OFF ) list(APPEND build_list raja ) @@ -470,6 +473,7 @@ ExternalProject_Add( chai -D ENABLE_CUDA:BOOL=${ENABLE_CUDA} -D CHAI_ENABLE_CUDA:BOOL=${ENABLE_CUDA} -D CMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER} + -D CMAKE_CUDA_HOST_COMPILER:FILEPATH=${CMAKE_CUDA_HOST_COMPILER} -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} From a4f1cdd6f0246e2b60a46c5e87e45b861c8e10d8 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 27 Aug 2026 22:58:50 -0400 Subject: [PATCH 04/22] Build Scotch with Makefiles --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7b4e7d6..f84068de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1255,9 +1255,10 @@ if( ENABLE_SCOTCH ) URL ${SCOTCH_URL} URL_HASH SHA256=${SCOTCH_URL_HASH} INSTALL_DIR ${SCOTCH_DIR} - BUILD_COMMAND ${TPL_BUILD_COMMAND} - INSTALL_COMMAND ${TPL_INSTALL_COMMAND} - CMAKE_GENERATOR ${TPL_GENERATOR} + BINARY_DIR ${PROJECT_BINARY_DIR}/scotch/src/scotch-make-build + BUILD_COMMAND make -j ${NUM_PROC} + INSTALL_COMMAND make install + CMAKE_GENERATOR "Unix Makefiles" CMAKE_ARGS -D CMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -D BUILD_PTSCOTCH:BOOL=ON -D BUILD_LIBSCOTCHMETIS:BOOL=OFF From ea43fa5bf60c71abdd71159a72ddcb5aaffdc4a0 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 27 Aug 2026 23:18:17 -0400 Subject: [PATCH 05/22] Propagate CUDA flags to HYPRE --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f84068de..82313e97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -966,6 +966,10 @@ if( ENABLE_HYPRE ) --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + if( NOT "${CMAKE_CUDA_FLAGS}" STREQUAL "" ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-extra-CUFLAGS=\"${CMAKE_CUDA_FLAGS}\"" ) + endif() + if( ENABLE_HYPRE_GPU_AWARE_MPI ) set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) endif() From ac91def3cad771ac48096f6f4752ad6a9e4adb41 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 27 Aug 2026 23:20:56 -0400 Subject: [PATCH 06/22] Add MPI includes to HYPRE CUDA builds --- CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82313e97..17bbd0f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -970,6 +970,19 @@ if( ENABLE_HYPRE ) set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-extra-CUFLAGS=\"${CMAKE_CUDA_FLAGS}\"" ) endif() + if( ENABLE_MPI ) + if( MPI_C_INCLUDE_DIRS ) + list( GET MPI_C_INCLUDE_DIRS 0 HYPRE_MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) + get_filename_component( HYPRE_MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( HYPRE_MPI_PREFIX_DIR ${HYPRE_MPI_BIN_DIR} DIRECTORY ) + set( HYPRE_MPI_INCLUDE_DIR "${HYPRE_MPI_PREFIX_DIR}/include" ) + endif() + if( HYPRE_MPI_INCLUDE_DIR ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-MPI-include=${HYPRE_MPI_INCLUDE_DIR}" ) + endif() + endif() + if( ENABLE_HYPRE_GPU_AWARE_MPI ) set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) endif() From cba69523c89d8e2db92a405d440d5ba9f6bb29ba Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 28 Aug 2026 23:51:12 -0400 Subject: [PATCH 07/22] Fix HIP RAJA configuration --- CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17bbd0f0..c75ff239 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -397,6 +397,19 @@ set(RAJA_URL_HASH "2774a569f19a7d8e37d1e166d197c828c6b614e7f8ef6d16ccbfb144b8284 message(STATUS "Building RAJA found at ${RAJA_URL}") +# RDNA GPUs use 32-lane wavefronts, while the CDNA GPUs targeted by the +# default RAJA configuration use 64-lane wavefronts. Keep this override +# configurable so callers can select the wave size explicitly when needed. +if( ENABLE_HIP ) + set( RAJA_HIP_WAVESIZE_DEFAULT 64 ) + if( CMAKE_HIP_ARCHITECTURES MATCHES "(^|;)gfx(10|11)" ) + set( RAJA_HIP_WAVESIZE_DEFAULT 32 ) + endif() + set( RAJA_HIP_WAVESIZE "${RAJA_HIP_WAVESIZE_DEFAULT}" CACHE STRING + "RAJA HIP wavefront size (32 for RDNA, 64 for CDNA)" ) + message( STATUS "RAJA HIP wavefront size = ${RAJA_HIP_WAVESIZE}" ) +endif() + # RAJA's vectorization layer does not compile with the ROCm toolchain. if( ENABLE_HIP ) set( RAJA_ENABLE_VECTORIZATION OFF ) @@ -434,6 +447,7 @@ ExternalProject_Add( raja -D hip_DIR=${HIP_ROOT}/lib/cmake/hip -D ROCPRIM_DIR=${HIP_ROOT} -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} + -D RAJA_HIP_WAVESIZE:STRING=${RAJA_HIP_WAVESIZE} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} From 67bfca74534197b9841a168b4751a1b44138860a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 29 Aug 2026 03:10:18 -0400 Subject: [PATCH 08/22] Fix HIP compiler path --- scripts/spack_packages/packages/geosx/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index de59ce3e..f912356f 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -482,7 +482,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+rocm' in spec: cfg.write(cmake_cache_option('ENABLE_HIP', True)) cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', spec.variants['cxxstd'].value)) - cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) + hip_compiler = pjoin(str(spec['hip'].prefix), 'bin', 'amdclang++') + cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', hip_compiler)) if not spec.satisfies('amdgpu_target=none'): cmake_hip_archs = ";".join(spec.variants["amdgpu_target"].value) From d92717b671cf228fe09d9763a10849e45bcae016 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 30 Aug 2026 19:23:40 -0400 Subject: [PATCH 09/22] Add perlmutter spack config --- scripts/spack_configs/perlmutter/spack.yaml | 303 ++++++++++++++++++ .../packages/cray-libsci/package.py | 73 +++++ .../packages/superlu-dist/package.py | 30 ++ 3 files changed, 406 insertions(+) create mode 100644 scripts/spack_configs/perlmutter/spack.yaml create mode 100644 scripts/spack_packages/packages/cray-libsci/package.py create mode 100644 scripts/spack_packages/packages/superlu-dist/package.py diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml new file mode 100644 index 00000000..65e5c194 --- /dev/null +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -0,0 +1,303 @@ +#------------------------------------------------------------------------------------------------------------ +# SPDX-License-Identifier: LGPL-2.1-only +# +# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC +# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University +# Copyright (c) 2018-2020 TotalEnergies +# Copyright (c) 2019- GEOSX Contributors +# All rights reserved +# +# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. +#------------------------------------------------------------------------------------------------------------ +# +# NERSC Perlmutter GPU (A100 / cuda_arch=80, Milan / znver3). +# Uses the default CPE 26.03 stack: gcc-native/14 or nvidia/26.5, cray-mpich/9.1.0, +# cray-libsci/26.03.0, and cudatoolkit/13.2. +# +# CFS install (private dir under project m4553, no nested quota): +# mkdir -p "$CFS/m4553/${USER}/geos-tpl" +# chmod 700 "$CFS/m4553/${USER}" +# +# Before uberenv, on a GPU allocation (-C gpu -A m4553_g): +# module unload darshan +# module load python/3.12-26.1.0 # /usr/bin/python3 is 3.6; too old for Spack +# # Stage on a FS that supports flock. CFS is DVS-mounted on compute nodes +# # (error 524 / autom4te). Prefer node-local NVMe; PSCRATCH is second choice. +# unset TMPDIR +# export TMPDIR=/tmp +# mkdir -p "$TMPDIR" +# +# GNU (recommended): +# module load PrgEnv-gnu/8.7.0 gcc-native/14 craype-accel-nvidia80 cudatoolkit/13.2 cmake/3.30.2 +# python3 ./scripts/uberenv/uberenv.py \ +# --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%gcc-14 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja" \ +# --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ +# --prefix "$CFS/m4553/${USER}/geos-tpl" +# +# NVHPC (do not load cudatoolkit; it conflicts with the nvidia module): +# module load PrgEnv-nvidia/8.7.0 nvidia/26.5 craype-accel-nvidia80 cmake/3.30.2 +# python3 ./scripts/uberenv/uberenv.py \ +# --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%nvhpc-26 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja" \ +# --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ +# --prefix "$CFS/m4553/${USER}/geos-tpl" +# +# NOTE: Use "%%" to propagate the compiler choice to dependency libraries. +# +# If the NVHPC cray-libsci prefix is wrong, run: +# module load PrgEnv-nvidia/8.7.0 nvidia/26.5 +# echo "$CRAY_LIBSCI_PREFIX_DIR" +# +# Resume a live env after gmp/superlu-dist failures (do not copy this file +# over spack_env/spack.yaml; only patch config:build_stage, then concretize): +# export TMPDIR=/tmp +# ENV="$CFS/m4553/${USER}/geos/tpl-install/spack_env" +# SPACK="$CFS/m4553/${USER}/geos/tpl-install/spack/bin/spack" +# "$SPACK" -D "$ENV" config add 'config:build_stage:[$tempdir/$user/spack-stage, $spack/../build_stage]' +# "$SPACK" -D "$ENV" concretize -f +# "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig +# + +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + # Do not stage on CFS: DVS has no flock, so gmp/mpfr autoreconf fails. + # $tempdir follows TMPDIR (/tmp on the GPU node, or $PSCRATCH if you set it). + build_stage: + - $tempdir/$user/spack-stage + - $spack/../build_stage + build_jobs: 32 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new + + # Regular TPLs do not need views + view: false + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + toolchains: + gcc-14: + - spec: '%c=gcc@14.3.0' + when: '%c' + - spec: '%cxx=gcc@14.3.0' + when: '%cxx' + - spec: '%fortran=gcc@14.3.0' + when: '%fortran' + - spec: '%cray-mpich@9.1.0_gnu' + when: '%mpi' + nvhpc-26: + - spec: '%c=nvhpc@26.5' + when: '%c' + - spec: '%cxx=nvhpc@26.5' + when: '%cxx' + - spec: '%fortran=nvhpc@26.5' + when: '%fortran' + - spec: '%cray-mpich@9.1.0_nvhpc' + when: '%mpi' + + packages: + all: + target: [znver3] + providers: + blas: [cray-libsci] + lapack: [cray-libsci] + mpi: [cray-mpich] + zlib-api: [zlib] + pkgconfig: [pkg-config] + + gcc: + buildable: false + externals: + - spec: gcc@14.3.0 languages:=c,c++,fortran + prefix: /opt/cray/pe/gcc-native/14 + extra_attributes: + compilers: + c: /opt/cray/pe/gcc-native/14/bin/gcc + cxx: /opt/cray/pe/gcc-native/14/bin/g++ + fortran: /opt/cray/pe/gcc-native/14/bin/gfortran + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/cray/pe/mpich/9.1.0/ofi/gnu/12.3/lib + - /opt/cray/pe/mpich/9.1.0/gtl/lib + - /opt/cray/pe/libsci/26.03.0/GNU/12/x86_64/lib + - /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64 + modules: + - PrgEnv-gnu/8.7.0 + - gcc-native/14 + - craype-x86-milan + - craype/2.7.36 + - craype-network-ofi + - craype-accel-nvidia80 + - cray-mpich/9.1.0 + - cray-libsci/26.03.0 + - cudatoolkit/13.2 + - cray-dsmml/0.3.1 + - libfabric/1.22.0 + - xpmem/2.11.5-1.3_g73ade43320bc + - cpe/26.03 + - cmake/3.30.2 + - perftools-base/26.03.0 + + nvhpc: + buildable: false + externals: + - spec: nvhpc@26.5 + prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5 + extra_attributes: + compilers: + c: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/bin/nvc++ + fortran: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/bin/nvfortran + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/cray/pe/mpich/9.1.0/ofi/nvidia/23.3/lib + - /opt/cray/pe/mpich/9.1.0/gtl/lib + - /opt/cray/pe/libsci/26.03.0/NVIDIA/23/x86_64/lib + - /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64 + - /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/lib + modules: + - PrgEnv-nvidia/8.7.0 + - nvidia/26.5 + - craype-x86-milan + - craype/2.7.36 + - craype-network-ofi + - craype-accel-nvidia80 + - cray-mpich/9.1.0 + - cray-libsci/26.03.0 + - cray-dsmml/0.3.1 + - libfabric/1.22.0 + - xpmem/2.11.5-1.3_g73ade43320bc + - cpe/26.03 + - cmake/3.30.2 + - perftools-base/26.03.0 + + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@9.1.0_gnu %gcc@14.3.0 + prefix: /opt/cray/pe/mpich/9.1.0/ofi/gnu/12.3 + - spec: cray-mpich@9.1.0_nvhpc %nvhpc@26.5 + prefix: /opt/cray/pe/mpich/9.1.0/ofi/nvidia/23.3 + + cray-libsci: + buildable: false + externals: + - spec: cray-libsci@26.03.0 %gcc@14.3.0 + prefix: /opt/cray/pe/libsci/26.03.0/GNU/12/x86_64 + # ABI dir matches cray-mpich nvidia/23.3. Confirm with $CRAY_LIBSCI_PREFIX_DIR. + - spec: cray-libsci@26.03.0 %nvhpc@26.5 + prefix: /opt/cray/pe/libsci/26.03.0/NVIDIA/23/x86_64 + + cuda: + buildable: false + externals: + - spec: cuda@13.2.0 +allow-unsupported-compilers + prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2 + + cmake: + buildable: false + externals: + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 + modules: + - cmake/3.30.2 + + ninja: + buildable: false + externals: + - spec: ninja@1.10.0 + prefix: /usr + + glibc: + buildable: false + externals: + - spec: glibc@2.38 + prefix: /usr + + autoconf: + buildable: false + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + buildable: false + externals: + - spec: automake@1.15.1 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + perl: + buildable: false + externals: + - spec: perl@5.26.1 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + gmake: + buildable: false + externals: + - spec: gmake@4.2.1 + prefix: /usr + git: + buildable: false + externals: + - spec: git@2.51.0 ~tcltk + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.34 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + curl: + buildable: false + externals: + - spec: curl@8.14.1 + prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + zlib: + buildable: false + externals: + - spec: zlib@1.2.13 + prefix: /usr + binutils: + buildable: false + externals: + - spec: binutils@2.45 + prefix: /usr + # System bison is 3.0.4 (too old for scotch). Let Spack build bison. + # Do not mark flex external; scotch has issues with some system flex. diff --git a/scripts/spack_packages/packages/cray-libsci/package.py b/scripts/spack_packages/packages/cray-libsci/package.py new file mode 100644 index 00000000..7081cad0 --- /dev/null +++ b/scripts/spack_packages/packages/cray-libsci/package.py @@ -0,0 +1,73 @@ +from glob import glob +import os +import re + +from spack.package import * +from spack_repo.builtin.packages.cray_libsci.package import CrayLibsci as BuiltinCrayLibsci + + +class CrayLibsci(BuiltinCrayLibsci): + # Builtin recipe only lists versions through 23.02 and looks for unversioned + # names (libsci_gnu.so). CPE 26.03 on Perlmutter ships compiler-ABI names + # (libsci_gnu_123.so, libsci_nvidia_*.so) and no unversioned symlink, so + # spec["blas"].libs is empty and CMake consumers pass -l-l / a space. + version("26.03.0") + + @property + def blas_libs(self): + try: + libs = super().blas_libs + except RuntimeError: + libs = None + if libs: + return libs + + candidates = [name for name in self.canonical_names.values() if name in self.prefix] + if len(candidates) != 1: + return libs if libs is not None else find_libraries( + ["libsci_missing"], root=str(self.prefix), shared=True, recursive=False + ) + tag = candidates[0].lower() + + if self.spec.satisfies("+openmp") and self.spec.satisfies("+mpi"): + suffixes = ["_mpi_mp", "_mp"] + elif self.spec.satisfies("+openmp"): + suffixes = ["_mp"] + elif self.spec.satisfies("+mpi"): + suffixes = ["_mpi", ""] + else: + suffixes = [""] + + shared = "+shared" in self.spec + lib_dirs = [p for p in (self.prefix.lib, self.prefix.lib64) if os.path.isdir(p)] + ext = "so" if shared else "a" + + def ver_key(path): + base = os.path.basename(path) + match = re.search(r"_(\d+)\.(?:so|a)", base) + compiler_abi = int(match.group(1)) if match else 0 + # Prefer the unversioned .so/.a symlink over libfoo.so.6. + is_devlink = 1 if re.search(r"\.(so|a)$", base) else 0 + return (compiler_abi, is_devlink) + + for suffix in suffixes: + rx = re.compile( + r"^libsci_{tag}(_\d+)?{suffix}\.(so|a)(\.\d+)*$".format( + tag=re.escape(tag), suffix=re.escape(suffix) + ) + ) + matches = [] + for lib_dir in lib_dirs: + for path in glob(os.path.join(lib_dir, f"libsci_{tag}*{suffix}.{ext}*")): + if rx.match(os.path.basename(path)): + matches.append(path) + if matches: + best = sorted(set(matches), key=ver_key)[-1] + libname = os.path.basename(best).split(".")[0] + return find_libraries( + [libname], root=os.path.dirname(best), shared=shared, recursive=False + ) + + return libs if libs is not None else find_libraries( + ["libsci_missing"], root=str(self.prefix), shared=True, recursive=False + ) diff --git a/scripts/spack_packages/packages/superlu-dist/package.py b/scripts/spack_packages/packages/superlu-dist/package.py new file mode 100644 index 00000000..1ac1c781 --- /dev/null +++ b/scripts/spack_packages/packages/superlu-dist/package.py @@ -0,0 +1,30 @@ +from spack.package import * +from spack_repo.builtin.packages.superlu_dist.package import SuperluDist as BuiltinSuperluDist + + +class SuperluDist(BuiltinSuperluDist): + # SuperLU_DIST's CMake wants absolute library paths in TPL_*_LIBRARIES. + # The builtin recipe passes ld_flags (-L/-l). When cray-libsci.libs is + # empty that becomes a blank, FindBLAS on Cray fills in "-l", and ld + # fails with "cannot find -l-l". + def cmake_args(self): + args = [ + a + for a in super().cmake_args() + if "TPL_BLAS_LIBRARIES" not in a and "TPL_LAPACK_LIBRARIES" not in a + ] + + def abs_libs(virtual): + libs = self.spec[virtual].libs + if not libs: + raise InstallError( + "{0} libraries were empty for {1}; " + "check the cray-libsci prefix in packages.yaml".format( + virtual, self.spec[virtual] + ) + ) + return libs.joined(";") + + args.append(self.define("TPL_BLAS_LIBRARIES", abs_libs("blas"))) + args.append(self.define("TPL_LAPACK_LIBRARIES", abs_libs("lapack"))) + return args From c708ce752d3784e72786e45f267a8ccfdc305110 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 30 Aug 2026 21:07:17 -0400 Subject: [PATCH 10/22] Fixes to perlmutter spack config --- scripts/spack_configs/perlmutter/spack.yaml | 14 +++++++++++++- .../packages/superlu-dist/package.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index 65e5c194..58697d6d 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -53,6 +53,8 @@ # ENV="$CFS/m4553/${USER}/geos/tpl-install/spack_env" # SPACK="$CFS/m4553/${USER}/geos/tpl-install/spack/bin/spack" # "$SPACK" -D "$ENV" config add 'config:build_stage:[$tempdir/$user/spack-stage, $spack/../build_stage]' +# "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' +# "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' # "$SPACK" -D "$ENV" concretize -f # "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig # @@ -112,6 +114,16 @@ spack: zlib-api: [zlib] pkgconfig: [pkg-config] + # nvhpc also provides blas/lapack (default +blas+lapack) and the concretizer + # will pick it over cray-libsci because the compiler is already in the DAG. + # Force LibSci; otherwise TPL_BLAS_LIBRARIES and SuiteSparse BLAS= are empty. + blas: + require: + - cray-libsci + lapack: + require: + - cray-libsci + gcc: buildable: false externals: @@ -152,7 +164,7 @@ spack: nvhpc: buildable: false externals: - - spec: nvhpc@26.5 + - spec: nvhpc@26.5 ~blas ~lapack prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5 extra_attributes: compilers: diff --git a/scripts/spack_packages/packages/superlu-dist/package.py b/scripts/spack_packages/packages/superlu-dist/package.py index 1ac1c781..e6c4ff94 100644 --- a/scripts/spack_packages/packages/superlu-dist/package.py +++ b/scripts/spack_packages/packages/superlu-dist/package.py @@ -19,7 +19,7 @@ def abs_libs(virtual): if not libs: raise InstallError( "{0} libraries were empty for {1}; " - "check the cray-libsci prefix in packages.yaml".format( + "expected cray-libsci (not nvhpc) as the {0} provider".format( virtual, self.spec[virtual] ) ) From cfeef09b4c8c5a91101e4b77147f9de9a38b084c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 30 Aug 2026 22:26:40 -0400 Subject: [PATCH 11/22] Fix nvhpc issues --- scripts/spack_configs/perlmutter/spack.yaml | 27 +++++++++++++++++-- .../spack_packages/packages/vtk/package.py | 15 +++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index 58697d6d..e47741ac 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -34,11 +34,14 @@ # --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ # --prefix "$CFS/m4553/${USER}/geos-tpl" # -# NVHPC (do not load cudatoolkit; it conflicts with the nvidia module): +# NVHPC (do not load cudatoolkit; it conflicts with the nvidia module). +# nvc cannot build GNU C/autotools (python, libffi, libiconv). Reuse gcc python. +# Use a separate env name so uberenv does not delete the gcc spack_env. # module load PrgEnv-nvidia/8.7.0 nvidia/26.5 craype-accel-nvidia80 cmake/3.30.2 # python3 ./scripts/uberenv/uberenv.py \ -# --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%nvhpc-26 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja" \ +# --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%nvhpc-26 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja ^python%gcc-14" \ # --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ +# --spack-env-name=spack_env_nvhpc \ # --prefix "$CFS/m4553/${USER}/geos-tpl" # # NOTE: Use "%%" to propagate the compiler choice to dependency libraries. @@ -58,6 +61,17 @@ # "$SPACK" -D "$ENV" concretize -f # "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig # +# NVHPC env (after git pull of vtk/zstd overlays). Reuse gcc python; force LibSci: +# export TMPDIR=/tmp +# ENV="$CFS/m4553/${USER}/geos/tpl-install/spack_env_nvhpc" +# SPACK="$CFS/m4553/${USER}/geos/tpl-install/spack/bin/spack" +# "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' +# "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' +# "$SPACK" -D "$ENV" config add 'packages:python:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'concretizer:unify:when_possible' +# "$SPACK" -D "$ENV" concretize -f +# "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig +# spack: config: @@ -76,6 +90,10 @@ spack: # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. installer: new + # Allow python%gcc next to %%nvhpc-26 (nvc cannot build libffi/libiconv/libmd). + concretizer: + unify: when_possible + # Regular TPLs do not need views view: false @@ -124,6 +142,11 @@ spack: require: - cray-libsci + # nvc 26.5 ICEs or miscompiles GNU C (libffi .S, libiconv error.h, libmd patch). + python: + require: + - "%gcc" + gcc: buildable: false externals: diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index 2a75d931..c67d00ff 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -71,6 +71,21 @@ class Vtk(CMakePackage): # not promote its expected condition to a compiler warning. patch('9.7.0-patch/diy-fortify-macro.patch', when='@9.7.0') + @run_before("cmake") + def nvhpc_workaround_surface_nets_ice(self): + # nvc++ 26.5 ICE: interpret.cpp extract_value_from_constant on this + # generated table file at default -O2. + if not self.spec.satisfies("%nvhpc"): + return + cmakelists = join_path(self.stage.source_path, "Filters", "Core", "CMakeLists.txt") + with open(cmakelists, "a") as handle: + handle.write( + "\n# nvc++ ICE on vtkSurfaceNets3DNonManifoldCases.cxx at -O2+\n" + "set_source_files_properties(\n" + " vtkSurfaceNets3DNonManifoldCases.cxx\n" + ' PROPERTIES COMPILE_FLAGS "-O0")\n' + ) + def cmake_args(self): spec = self.spec From 1e9b1f1f49b1c864f1db69868e81f1c65dbe1f9b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 30 Aug 2026 23:11:15 -0400 Subject: [PATCH 12/22] Fix nvhpc issues --- scripts/spack_configs/perlmutter/spack.yaml | 32 ++++++++++++++++--- .../spack_packages/packages/geosx/package.py | 5 ++- .../spack_packages/packages/vtk/package.py | 18 +++-------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index e47741ac..8d15668b 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -35,7 +35,8 @@ # --prefix "$CFS/m4553/${USER}/geos-tpl" # # NVHPC (do not load cudatoolkit; it conflicts with the nvidia module). -# nvc cannot build GNU C/autotools (python, libffi, libiconv). Reuse gcc python. +# Keep "%%" so C++ TPLs (vtk, raja, hypre) stay nvc. %% is a preference; +# packages.yaml require %gcc is a hard exception for GNU C that nvc 26.5 ICEs. # Use a separate env name so uberenv does not delete the gcc spack_env. # module load PrgEnv-nvidia/8.7.0 nvidia/26.5 craype-accel-nvidia80 cmake/3.30.2 # python3 ./scripts/uberenv/uberenv.py \ @@ -61,15 +62,25 @@ # "$SPACK" -D "$ENV" concretize -f # "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig # -# NVHPC env (after git pull of vtk/zstd overlays). Reuse gcc python; force LibSci: +# NVHPC env. Copy overlays into the geos-tpls checkout (zstd/libmd are not +# on GitHub until committed). Same hashes as a previous fail mean the lockfile +# was not re-concretized — config add on a live env does not restage vtk. # export TMPDIR=/tmp # ENV="$CFS/m4553/${USER}/geos/tpl-install/spack_env_nvhpc" # SPACK="$CFS/m4553/${USER}/geos/tpl-install/spack/bin/spack" # "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' # "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' # "$SPACK" -D "$ENV" config add 'packages:python:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:libffi:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:libiconv:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:libmd:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:zstd:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'concretizer:unify:when_possible' # "$SPACK" -D "$ENV" concretize -f +# "$SPACK" -D "$ENV" spec -I geosx | grep -E 'python|libffi|libiconv|libmd |zstd|vtk' +# # python/libffi/libiconv/libmd/zstd must be %gcc@14.3.0 under gcc-14.3.0/ +# # (new hashes). vtk stays %nvhpc@26.5 with a NEW hash (surface-nets patch). +# "$SPACK" -D "$ENV" uninstall --yes /fnqnbm7 /4e3655z /auvwsmg /jdknojp /xqymogj || true # "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig # @@ -90,7 +101,7 @@ spack: # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. installer: new - # Allow python%gcc next to %%nvhpc-26 (nvc cannot build libffi/libiconv/libmd). + # Allow python%gcc next to %%nvhpc-26. %% is a preference; require wins. concretizer: unify: when_possible @@ -142,10 +153,23 @@ spack: require: - cray-libsci - # nvc 26.5 ICEs or miscompiles GNU C (libffi .S, libiconv error.h, libmd patch). + # nvc 26.5 ICEs GNU C (libffi .S, libiconv error.h, libmd aliases, zstd -MF). + # Hard exception to %%nvhpc-26. After concretize these must not be %nvhpc. python: require: - "%gcc" + libffi: + require: + - "%gcc" + libiconv: + require: + - "%gcc" + libmd: + require: + - "%gcc" + zstd: + require: + - "%gcc" gcc: buildable: false diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index f912356f..7f57cbfc 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -208,7 +208,10 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Python # - depends_on('python') + # Hostconfig always needs an interpreter. Keep python off the link/run DAG + # unless +pygeosx; otherwise %%nvhpc prefers nvc for libffi/libiconv/libmd. + depends_on('python', type='build', when='~pygeosx') + depends_on('python', type=('build', 'link', 'run'), when='+pygeosx') # diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index c67d00ff..63196979 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -71,20 +71,10 @@ class Vtk(CMakePackage): # not promote its expected condition to a compiler warning. patch('9.7.0-patch/diy-fortify-macro.patch', when='@9.7.0') - @run_before("cmake") - def nvhpc_workaround_surface_nets_ice(self): - # nvc++ 26.5 ICE: interpret.cpp extract_value_from_constant on this - # generated table file at default -O2. - if not self.spec.satisfies("%nvhpc"): - return - cmakelists = join_path(self.stage.source_path, "Filters", "Core", "CMakeLists.txt") - with open(cmakelists, "a") as handle: - handle.write( - "\n# nvc++ ICE on vtkSurfaceNets3DNonManifoldCases.cxx at -O2+\n" - "set_source_files_properties(\n" - " vtkSurfaceNets3DNonManifoldCases.cxx\n" - ' PROPERTIES COMPILE_FLAGS "-O0")\n' - ) + # nvc++ 26.5 ICE: interpret.cpp extract_value_from_constant on the + # constexpr SurfaceNets tables. A run_before("cmake") edit is skipped on + # --keep-stage retries and does not change the spec hash. + patch('nvhpc-surface-nets-ice.patch', when='@9.7.0 %nvhpc') def cmake_args(self): spec = self.spec From cf1b84dd1e9535a9f9533bcfceddbe47c10515cb Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 30 Aug 2026 23:12:29 -0400 Subject: [PATCH 13/22] Fix nvhpc issues --- .../spack_packages/packages/libmd/package.py | 8 +++++ .../packages/vtk/nvhpc-surface-nets-ice.patch | 34 ++++++++++++++++++ .../spack_packages/packages/zstd/package.py | 36 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 scripts/spack_packages/packages/libmd/package.py create mode 100644 scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch create mode 100644 scripts/spack_packages/packages/zstd/package.py diff --git a/scripts/spack_packages/packages/libmd/package.py b/scripts/spack_packages/packages/libmd/package.py new file mode 100644 index 00000000..132c2a5a --- /dev/null +++ b/scripts/spack_packages/packages/libmd/package.py @@ -0,0 +1,8 @@ +from spack.package import * +from spack_repo.builtin.packages.libmd.package import Libmd as BuiltinLibmd + + +class Libmd(BuiltinLibmd): + # Builtin nvhpc-aliases.patch matches libmd_alias(); 1.1.0 renamed that + # to libmd_strong_alias(), so the hunk fails. nvc 26 supports aliases. + patches = [] diff --git a/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch b/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch new file mode 100644 index 00000000..d98d8c17 --- /dev/null +++ b/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch @@ -0,0 +1,34 @@ +--- a/Filters/Core/vtkSurfaceNets3DNonManifoldCases.cxx ++++ b/Filters/Core/vtkSurfaceNets3DNonManifoldCases.cxx +@@ -6016,10 +6016,17 @@ + // clang-format on + return { offsets, metadata }; + } ++#ifdef __NVCOMPILER ++// nvc++ 26.5 ICE in extract_value_from_constant on these constexpr tables. ++static const auto Result = CreateNonManifoldMetaDataPerEdgeCase(); ++static const auto& Offsets = Result.first; ++static const auto& Metadata = Result.second; ++#else + constexpr auto Result = CreateNonManifoldMetaDataPerEdgeCase(); + constexpr auto& Offsets = Result.first; + constexpr auto& Metadata = Result.second; + #endif ++#endif + VTK_ABI_NAMESPACE_END + } + +--- a/Filters/Core/CMakeLists.txt ++++ b/Filters/Core/CMakeLists.txt +@@ -152,6 +152,11 @@ + + vtk_add_test_mangling(VTK::FiltersCore) + ++if(CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") ++ # Keep this generated table at -O0; -fast -O3 ICEs nvc++ 26.5. ++ set_source_files_properties(vtkSurfaceNets3DNonManifoldCases.cxx PROPERTIES COMPILE_OPTIONS "-O0") ++endif() ++ + # Add precompiled header to improve build performance + if(VTK_USE_PCH) + vtk_module_precompile_headers(VTK::FiltersCore PRIVATE "$<$:${CMAKE_CURRENT_SOURCE_DIR}/vtkFiltersCorePCH.h>") diff --git a/scripts/spack_packages/packages/zstd/package.py b/scripts/spack_packages/packages/zstd/package.py new file mode 100644 index 00000000..ab00fb63 --- /dev/null +++ b/scripts/spack_packages/packages/zstd/package.py @@ -0,0 +1,36 @@ +from spack.package import * +from spack_repo.builtin.packages.zstd.package import MakefileBuilder as BuiltinZstdMakefileBuilder +from spack_repo.builtin.packages.zstd.package import Zstd as BuiltinZstd + + +class Zstd(BuiltinZstd): + pass + + +class MakefileBuilder(BuiltinZstdMakefileBuilder): + def install(self, pkg, spec, prefix): + args = ["VERBOSE=1", "PREFIX=" + prefix] + + # Builtin uses DEPFLAGS=-MT $@ -MMD -MF for %nvhpc (no -MP). nvc 26.x + # still requires -MF ; the makefile then passes -o, which nvc + # treats as a missing -MF argument. + if spec.satisfies("%nvhpc"): + args.append("DEPFLAGS=") + + lib_args = ["-C", "lib"] + args + ["install-pc", "install-includes"] + if "libs=shared" in spec: + lib_args.append("install-shared") + if "libs=static" in spec: + lib_args.append("install-static") + make(*lib_args) + + if "+programs" in spec: + programs_args = ["-C", "programs"] + args + if "compression=zlib" not in spec: + programs_args.append("HAVE_ZLIB=0") + if "compression=lzma" not in spec: + programs_args.append("HAVE_LZMA=0") + if "compression=lz4" not in spec: + programs_args.append("HAVE_LZ4=0") + programs_args.append("install") + make(*programs_args) From 26bd1fffe771cbd31b9f9b1f87db83bbc30e3f04 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 30 Aug 2026 23:30:29 -0400 Subject: [PATCH 14/22] Fix nvhpc issues --- scripts/spack_configs/perlmutter/spack.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index 8d15668b..fe3f1a34 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -14,11 +14,11 @@ # Uses the default CPE 26.03 stack: gcc-native/14 or nvidia/26.5, cray-mpich/9.1.0, # cray-libsci/26.03.0, and cudatoolkit/13.2. # -# CFS install (private dir under project m4553, no nested quota): -# mkdir -p "$CFS/m4553/${USER}/geos-tpl" -# chmod 700 "$CFS/m4553/${USER}" +# CFS install (private dir under project mXXXX, no nested quota): +# mkdir -p "$CFS/mXXXX/${USER}/geos-tpl" +# chmod 700 "$CFS/mXXXX/${USER}" # -# Before uberenv, on a GPU allocation (-C gpu -A m4553_g): +# Before uberenv, on a GPU allocation (-C gpu -A mXXXX_g): # module unload darshan # module load python/3.12-26.1.0 # /usr/bin/python3 is 3.6; too old for Spack # # Stage on a FS that supports flock. CFS is DVS-mounted on compute nodes @@ -32,7 +32,7 @@ # python3 ./scripts/uberenv/uberenv.py \ # --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%gcc-14 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja" \ # --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ -# --prefix "$CFS/m4553/${USER}/geos-tpl" +# --prefix "$CFS/mXXXX/${USER}/geos-tpl" # # NVHPC (do not load cudatoolkit; it conflicts with the nvidia module). # Keep "%%" so C++ TPLs (vtk, raja, hypre) stay nvc. %% is a preference; @@ -43,7 +43,7 @@ # --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%nvhpc-26 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja ^python%gcc-14" \ # --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ # --spack-env-name=spack_env_nvhpc \ -# --prefix "$CFS/m4553/${USER}/geos-tpl" +# --prefix "$CFS/mXXXX/${USER}/geos-tpl" # # NOTE: Use "%%" to propagate the compiler choice to dependency libraries. # @@ -54,8 +54,8 @@ # Resume a live env after gmp/superlu-dist failures (do not copy this file # over spack_env/spack.yaml; only patch config:build_stage, then concretize): # export TMPDIR=/tmp -# ENV="$CFS/m4553/${USER}/geos/tpl-install/spack_env" -# SPACK="$CFS/m4553/${USER}/geos/tpl-install/spack/bin/spack" +# ENV="$CFS/mXXXX/${USER}/geos/tpl-install/spack_env" +# SPACK="$CFS/mXXXX/${USER}/geos/tpl-install/spack/bin/spack" # "$SPACK" -D "$ENV" config add 'config:build_stage:[$tempdir/$user/spack-stage, $spack/../build_stage]' # "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' # "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' @@ -66,8 +66,8 @@ # on GitHub until committed). Same hashes as a previous fail mean the lockfile # was not re-concretized — config add on a live env does not restage vtk. # export TMPDIR=/tmp -# ENV="$CFS/m4553/${USER}/geos/tpl-install/spack_env_nvhpc" -# SPACK="$CFS/m4553/${USER}/geos/tpl-install/spack/bin/spack" +# ENV="$CFS/mXXXX/${USER}/geos/tpl-install/spack_env_nvhpc" +# SPACK="$CFS/mXXXX/${USER}/geos/tpl-install/spack/bin/spack" # "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' # "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' # "$SPACK" -D "$ENV" config add 'packages:python:require:["%gcc"]' From 9379647d84c127bb6f60b4adc06e6880db0fc8d8 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 31 Aug 2026 00:06:16 -0400 Subject: [PATCH 15/22] Fix nvhpc issues --- scripts/spack_configs/perlmutter/spack.yaml | 36 +++++++++++++++++-- .../packages/vtk/nvhpc-surface-nets-ice.patch | 17 +++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index fe3f1a34..57a9f7fc 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -75,6 +75,14 @@ # "$SPACK" -D "$ENV" config add 'packages:libiconv:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'packages:libmd:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'packages:zstd:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:gettext:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:expat:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:gdbm:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:ncurses:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:readline:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:sqlite:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:util-linux-uuid:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:texinfo:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'concretizer:unify:when_possible' # "$SPACK" -D "$ENV" concretize -f # "$SPACK" -D "$ENV" spec -I geosx | grep -E 'python|libffi|libiconv|libmd |zstd|vtk' @@ -153,8 +161,8 @@ spack: require: - cray-libsci - # nvc 26.5 ICEs GNU C (libffi .S, libiconv error.h, libmd aliases, zstd -MF). - # Hard exception to %%nvhpc-26. After concretize these must not be %nvhpc. + # nvc 26.5 ICEs GNU C (libffi .S, libiconv error.h, libmd aliases, zstd -MF, + # gettext nvhpc-export-symbols.patch vs 1.0). Hard exception to %%nvhpc-26. python: require: - "%gcc" @@ -170,6 +178,30 @@ spack: zstd: require: - "%gcc" + gettext: + require: + - "%gcc" + expat: + require: + - "%gcc" + gdbm: + require: + - "%gcc" + ncurses: + require: + - "%gcc" + readline: + require: + - "%gcc" + sqlite: + require: + - "%gcc" + util-linux-uuid: + require: + - "%gcc" + texinfo: + require: + - "%gcc" gcc: buildable: false diff --git a/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch b/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch index d98d8c17..4ef6436f 100644 --- a/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch +++ b/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch @@ -1,11 +1,23 @@ --- a/Filters/Core/vtkSurfaceNets3DNonManifoldCases.cxx +++ b/Filters/Core/vtkSurfaceNets3DNonManifoldCases.cxx -@@ -6016,10 +6016,17 @@ +@@ -8,6 +8,12 @@ + #include + #include + ++#ifdef __NVCOMPILER ++// nvc++ 26.5 ICE in extract_value_from_constant even at -O0 while evaluating ++// these constexpr tables. Drop constexpr so initialization is runtime. ++#define constexpr ++#endif ++ + // enable when creating the metadata table + // #define VTK_SURFACE_NETS_3D_NON_MANIFOLD_CASES_CREATE_TABLE + #ifdef VTK_SURFACE_NETS_3D_NON_MANIFOLD_CASES_CREATE_TABLE +@@ -6016,10 +6022,17 @@ // clang-format on return { offsets, metadata }; } +#ifdef __NVCOMPILER -+// nvc++ 26.5 ICE in extract_value_from_constant on these constexpr tables. +static const auto Result = CreateNonManifoldMetaDataPerEdgeCase(); +static const auto& Offsets = Result.first; +static const auto& Metadata = Result.second; @@ -25,7 +37,6 @@ vtk_add_test_mangling(VTK::FiltersCore) +if(CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") -+ # Keep this generated table at -O0; -fast -O3 ICEs nvc++ 26.5. + set_source_files_properties(vtkSurfaceNets3DNonManifoldCases.cxx PROPERTIES COMPILE_OPTIONS "-O0") +endif() + From 39e68b7b942ccc7ab8464d42441acc8871e3a358 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 31 Aug 2026 00:25:39 -0400 Subject: [PATCH 16/22] Fix nvhpc issues --- scripts/spack_configs/perlmutter/spack.yaml | 4 ++- .../packages/gettext/package.py | 25 +++++++++++++++++++ .../spack_packages/packages/vtk/package.py | 10 +++++--- 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 scripts/spack_packages/packages/gettext/package.py diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index 57a9f7fc..eef01c6a 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -84,7 +84,9 @@ # "$SPACK" -D "$ENV" config add 'packages:util-linux-uuid:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'packages:texinfo:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'concretizer:unify:when_possible' -# "$SPACK" -D "$ENV" concretize -f +# # Patch file changes require --fresh; -f alone reuses the lockfile vtk hash. +# "$SPACK" clean -m +# "$SPACK" -D "$ENV" concretize -f --fresh # "$SPACK" -D "$ENV" spec -I geosx | grep -E 'python|libffi|libiconv|libmd |zstd|vtk' # # python/libffi/libiconv/libmd/zstd must be %gcc@14.3.0 under gcc-14.3.0/ # # (new hashes). vtk stays %nvhpc@26.5 with a NEW hash (surface-nets patch). diff --git a/scripts/spack_packages/packages/gettext/package.py b/scripts/spack_packages/packages/gettext/package.py new file mode 100644 index 00000000..b233b2cb --- /dev/null +++ b/scripts/spack_packages/packages/gettext/package.py @@ -0,0 +1,25 @@ +from spack.package import * +from spack_repo.builtin.packages.gettext.package import Gettext as BuiltinGettext + + +class Gettext(BuiltinGettext): + # Builtin nvhpc-export-symbols.patch targets gettext-tools/intl (0.21 layout). + # 1.0 moved those files; the hunk fails. nvc is not used for gettext on + # Perlmutter (packages.yaml require %gcc). + pass + + +def _drop_nvhpc_export_symbols_patch(cls): + patches = getattr(cls, "patches", None) + if not patches: + return + kept = [] + for patch in patches: + path = getattr(patch, "relative_path", None) or getattr(patch, "path", None) or str(patch) + if "nvhpc-export-symbols" in str(path): + continue + kept.append(patch) + cls.patches = kept + + +_drop_nvhpc_export_symbols_patch(Gettext) diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index 63196979..45d9fcc7 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -72,9 +72,13 @@ class Vtk(CMakePackage): patch('9.7.0-patch/diy-fortify-macro.patch', when='@9.7.0') # nvc++ 26.5 ICE: interpret.cpp extract_value_from_constant on the - # constexpr SurfaceNets tables. A run_before("cmake") edit is skipped on - # --keep-stage retries and does not change the spec hash. - patch('nvhpc-surface-nets-ice.patch', when='@9.7.0 %nvhpc') + # constexpr SurfaceNets tables. sha256 must match the file; changing the + # patch without --fresh leaves the lockfile on the old hash (7671f62). + patch( + "nvhpc-surface-nets-ice.patch", + sha256="f56d3979fbd1b5b344c4dd7daf660ba75071f4503d447d12548a2c09102b54cc", + when="@9.7.0 %nvhpc", + ) def cmake_args(self): spec = self.spec From a9a2531c2214602f3388fb772da9192ce9a762bb Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 31 Aug 2026 00:43:26 -0400 Subject: [PATCH 17/22] Fix nvhpc issues --- scripts/spack_configs/perlmutter/spack.yaml | 8 +++---- .../packages/gettext/package.py | 24 +++++++++++++------ .../spack_packages/packages/libmd/package.py | 4 +++- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml index eef01c6a..2b0d7b0b 100644 --- a/scripts/spack_configs/perlmutter/spack.yaml +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -84,10 +84,10 @@ # "$SPACK" -D "$ENV" config add 'packages:util-linux-uuid:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'packages:texinfo:require:["%gcc"]' # "$SPACK" -D "$ENV" config add 'concretizer:unify:when_possible' -# # Patch file changes require --fresh; -f alone reuses the lockfile vtk hash. -# "$SPACK" clean -m -# "$SPACK" -D "$ENV" concretize -f --fresh -# "$SPACK" -D "$ENV" spec -I geosx | grep -E 'python|libffi|libiconv|libmd |zstd|vtk' +# # Do not use --fresh on this env (Spack 1.2.2). If vtk stays on the old +# # lockfile hash, back up and drop spack.lock, then concretize -f. +# "$SPACK" -D "$ENV" concretize -f +# "$SPACK" -D "$ENV" spec | grep -E 'python|libffi|libiconv|libmd |zstd|vtk' # # python/libffi/libiconv/libmd/zstd must be %gcc@14.3.0 under gcc-14.3.0/ # # (new hashes). vtk stays %nvhpc@26.5 with a NEW hash (surface-nets patch). # "$SPACK" -D "$ENV" uninstall --yes /fnqnbm7 /4e3655z /auvwsmg /jdknojp /xqymogj || true diff --git a/scripts/spack_packages/packages/gettext/package.py b/scripts/spack_packages/packages/gettext/package.py index b233b2cb..d833aa54 100644 --- a/scripts/spack_packages/packages/gettext/package.py +++ b/scripts/spack_packages/packages/gettext/package.py @@ -10,15 +10,25 @@ class Gettext(BuiltinGettext): def _drop_nvhpc_export_symbols_patch(cls): + # Spack stores patches as {when_spec: [patch, ...]}. Replacing that dict + # with a list makes concretize raise "'list' object has no attribute 'items'". patches = getattr(cls, "patches", None) - if not patches: + if not isinstance(patches, dict): return - kept = [] - for patch in patches: - path = getattr(patch, "relative_path", None) or getattr(patch, "path", None) or str(patch) - if "nvhpc-export-symbols" in str(path): - continue - kept.append(patch) + kept = {} + for when, plist in patches.items(): + filtered = [] + for patch in plist: + path = ( + getattr(patch, "relative_path", None) + or getattr(patch, "path", None) + or str(patch) + ) + if "nvhpc-export-symbols" in str(path): + continue + filtered.append(patch) + if filtered: + kept[when] = filtered cls.patches = kept diff --git a/scripts/spack_packages/packages/libmd/package.py b/scripts/spack_packages/packages/libmd/package.py index 132c2a5a..f65815c3 100644 --- a/scripts/spack_packages/packages/libmd/package.py +++ b/scripts/spack_packages/packages/libmd/package.py @@ -5,4 +5,6 @@ class Libmd(BuiltinLibmd): # Builtin nvhpc-aliases.patch matches libmd_alias(); 1.1.0 renamed that # to libmd_strong_alias(), so the hunk fails. nvc 26 supports aliases. - patches = [] + # Must be a dict (when_spec -> [patch]); a list breaks Spack 1.2 concretize + # with "'list' object has no attribute 'items'". + patches = {} From dd2cd02405b4e6bd638fd39ff044310938eb7a44 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 31 Aug 2026 17:27:09 -0400 Subject: [PATCH 18/22] Update hypre versions --- CMakeLists.txt | 8 ++++---- scripts/spack_packages/packages/hypre/package.py | 12 ++++++++++++ .../spack_packages/packages/hypredrive/package.py | 4 +++- 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 scripts/spack_packages/packages/hypre/package.py diff --git a/CMakeLists.txt b/CMakeLists.txt index c75ff239..706db60b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -928,8 +928,8 @@ endif() if( ENABLE_HYPRE ) set( HYPRE_DIR "${CMAKE_INSTALL_PREFIX}/hypre" ) - set( HYPRE_URL "https://github.com/hypre-space/hypre/archive/f1374fb6182c9e730abaa82f865a87b36f9ad50a.tar.gz" ) - set( HYPRE_URL_HASH "8b3c9176e9a4e3bc3e526d497aa52e5adbfbdff8810b90e9cc7413e898de6524" ) + set( HYPRE_URL "https://github.com/hypre-space/hypre/archive/53927ee4fb68e5caf6be1957af2c3d75905b6b3d.tar.gz" ) + set( HYPRE_URL_HASH "d418846bc1466616cc22e71c4d3b855127846307bb4f4a2f4d37f046ac9c3355" ) set( HYPRE_DEPENDS "" ) @@ -1094,8 +1094,8 @@ if( ENABLE_HYPREDRV ) endif() set( HYPREDRV_DIR "${CMAKE_INSTALL_PREFIX}/hypredrive" ) - set( HYPREDRV_URL "https://github.com/hypre-space/hypredrive/archive/98989bef31e865d738c1b678ee058bb0e5dd635e.tar.gz" ) - set( HYPREDRV_URL_HASH "7ea64b3ea5330dff2322bd265d22b2daded75733cdde8a6e5047feeea57b2cb5" ) + set( HYPREDRV_URL "https://github.com/hypre-space/hypredrive/archive/00cdb36800d6754b01467a4ef62e0e4ae19e6897.tar.gz" ) + set( HYPREDRV_URL_HASH "c33212d5fb7391c81ae22ae801d387f41c2c3f293f48ccd8153845d97ba5150c" ) set( HYPREDRV_C_COMPILER ${CMAKE_C_COMPILER} ) set( HYPREDRV_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" ) set( HYPREDRV_HYPRE_LIBRARY "${HYPRE_DIR}/lib/libHYPRE.a" ) diff --git a/scripts/spack_packages/packages/hypre/package.py b/scripts/spack_packages/packages/hypre/package.py new file mode 100644 index 00000000..1b29b44b --- /dev/null +++ b/scripts/spack_packages/packages/hypre/package.py @@ -0,0 +1,12 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * +from spack_repo.builtin.packages.hypre.package import Hypre as BuiltinHypre + + +class Hypre(BuiltinHypre): + # Keep the complete Git history so HYPRE can report its tagged, + # distance-aware development version (for example, v3.0.2-2-g). + version("develop", branch="master", get_full_repo=True) diff --git a/scripts/spack_packages/packages/hypredrive/package.py b/scripts/spack_packages/packages/hypredrive/package.py index 715e9349..e5a68e1b 100644 --- a/scripts/spack_packages/packages/hypredrive/package.py +++ b/scripts/spack_packages/packages/hypredrive/package.py @@ -23,7 +23,9 @@ class Hypredrive(CMakePackage, CudaPackage, ROCmPackage): license("MIT", checked_by="victorapm") - version("develop", branch="master") + # Keep the complete Git history so CMake can compute the tagged, + # distance-aware development version (for example, v0.2.0-2-g). + version("develop", branch="master", get_full_repo=True) version("0.2.0", sha256="2fe6c5b2779de41fbd294cb4647c7bbd210ec95934639117e56a790e56c32e41") version("0.1.0", sha256="39db73b75e37457035c64b4c8831abe716bf2f596c4ca79a32293d9bd51ca8d6") From e016855f2250f2e6747e4c00fa5f61d13d8d34f9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 31 Aug 2026 22:19:01 -0400 Subject: [PATCH 19/22] Update hypre install method + several fixes --- CMakeLists.txt | 526 ++++++++++++++++++++++++-------- cmake/geos-host-config.cmake.in | 116 +++++++ 2 files changed, 520 insertions(+), 122 deletions(-) create mode 100644 cmake/geos-host-config.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 706db60b..1d825cf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,7 @@ cmake_minimum_required(VERSION 3.23.1) +if( POLICY CMP0135 ) + cmake_policy( SET CMP0135 NEW ) +endif() ################################ # GEOS_TPL ################################ @@ -15,8 +18,23 @@ set( ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE ) ################################ # BLT ################################ -set( CXX_STANDARD 17 ) -set( BLT_CXX_STD c++17 CACHE STRING "" FORCE ) +set( CXX_STANDARD 20 ) +set( BLT_CXX_STD c++20 CACHE STRING "" FORCE ) + +# CUDA 13.3 rejects newer host compiler versions unless this opt-in is given. +# Keep it in the common CUDA flags so CMake/BLT and the external CUDA TPLs use +# the same NVCC behavior. +if( ENABLE_CUDA ) + string( FIND "${CMAKE_CUDA_FLAGS}" "--allow-unsupported-compiler" _GEOS_TPL_CUDA_FLAG_POS ) + if( _GEOS_TPL_CUDA_FLAG_POS EQUAL -1 ) + if( CMAKE_CUDA_FLAGS ) + set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --allow-unsupported-compiler" CACHE STRING "" FORCE ) + else() + set( CMAKE_CUDA_FLAGS "--allow-unsupported-compiler" CACHE STRING "" FORCE ) + endif() + endif() + unset( _GEOS_TPL_CUDA_FLAG_POS ) +endif() if (DEFINED BLT_SOURCE_DIR) # Support having a shared BLT outside of the repository if given a BLT_SOURCE_DIR @@ -149,8 +167,8 @@ if( ENABLE_HIP ) message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) endif() - # hypre is configured outside of cmake, so it needs the MPI include directory explicitly - # when its sources are compiled for the device. + # Device-enabled HYPRE/HypreDrive builds need the MPI include directory + # explicitly when their sources are compiled for the device. if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) @@ -417,6 +435,11 @@ else() set( RAJA_ENABLE_VECTORIZATION ON ) endif() +if( ENABLE_CUDA ) + set( RAJA_CUDA_FLAGS "${RAJA_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS}" ) + set( CHAI_CUDA_FLAGS "${CHAI_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS}" ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} URL_HASH SHA256=${RAJA_URL_HASH} @@ -932,153 +955,191 @@ if( ENABLE_HYPRE ) set( HYPRE_URL_HASH "d418846bc1466616cc22e71c4d3b855127846307bb4f4a2f4d37f046ac9c3355" ) set( HYPRE_DEPENDS "" ) - message(STATUS "Building HYPRE found at ${HYPRE_URL}") - set( HYPRE_C_COMPILER ${CMAKE_C_COMPILER} ) - set( HYPRE_CXX_COMPILER ${CMAKE_CXX_COMPILER} ) - set( HYPRE_Fortran_COMPILER ${CMAKE_Fortran_COMPILER} ) - set( HYPRE_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" ) - set( HYPRE_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}" ) - set( HYPRE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_${BUILD_TYPE_UPPER}}" ) + # HYPRE's CMake project is rooted in its src directory. Keep the HYPRE + # build tree separate from the extracted source tree; in-source CMake + # builds are not supported by HYPRE. + set( HYPRE_CMAKE_ARGS + "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" + "-DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}" + "-DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}" + "-DCMAKE_C_STANDARD:STRING=99" + "-DCMAKE_C_STANDARD_REQUIRED:BOOL=ON" + "-DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD}" + "-DCMAKE_CXX_STANDARD_REQUIRED:BOOL=ON" + "-DCMAKE_C_FLAGS:STRING=-fPIC ${C_FLAGS_NO_WARNINGS}" + "-DCMAKE_CXX_FLAGS:STRING=-fPIC ${CXX_FLAGS_NO_WARNINGS}" + "-DCMAKE_C_FLAGS_${BUILD_TYPE_UPPER}:STRING=${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" + "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${CMAKE_POSITION_INDEPENDENT_CODE}" + "-DCMAKE_INSTALL_PREFIX:PATH=" + "-DCMAKE_INSTALL_LIBDIR:STRING=lib" + "-DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}" + "-DHYPRE_ENABLE_MPI:BOOL=${ENABLE_MPI}" + "-DHYPRE_ENABLE_OPENMP:BOOL=${ENABLE_OPENMP}" + "-DHYPRE_ENABLE_FORTRAN:BOOL=OFF" + "-DHYPRE_ENABLE_PRINT_ERRORS:BOOL=ON" + "-DHYPRE_BUILD_EXAMPLES:BOOL=OFF" + "-DHYPRE_BUILD_TESTS:BOOL=OFF" + "-DHYPRE_ENABLE_HYPRE_BLAS:BOOL=OFF" + "-DHYPRE_ENABLE_HYPRE_LAPACK:BOOL=OFF" + "-DHYPRE_ENABLE_CUDA:BOOL=OFF" + "-DHYPRE_ENABLE_HIP:BOOL=OFF" + "-DHYPRE_ENABLE_BIGINT:BOOL=OFF" + "-DHYPRE_ENABLE_MIXEDINT:BOOL=OFF" + "-DHYPRE_ENABLE_UNIFIED_MEMORY:BOOL=OFF" + "-DHYPRE_ENABLE_UMPIRE:BOOL=OFF" + "-DHYPRE_ENABLE_DSUPERLU:BOOL=OFF" + "-DHYPRE_ENABLE_CALIPER:BOOL=OFF" + "-DHYPRE_ENABLE_GPU_AWARE_MPI:BOOL=OFF" + "-DHYPRE_ENABLE_GPU_PROFILING:BOOL=OFF" ) if( ENABLE_MPI ) - set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) - set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) - set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + # Use the host compilers for the HYPRE project. FindMPI is given the + # wrappers separately, which is important for CUDA: HYPRE's CUDA setup + # uses CMAKE_CXX_COMPILER as nvcc's host compiler. + list( APPEND HYPRE_CMAKE_ARGS + "-DMPI_C_COMPILER:FILEPATH=${MPI_C_COMPILER}" + "-DMPI_CXX_COMPILER:FILEPATH=${MPI_CXX_COMPILER}" ) endif() if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) - set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) - if( ENABLE_OPENMP ) - set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? - endif() - set( HYPRE_SUPERLU_FLAGS "--with-dsuperlu --with-dsuperlu-include=${SUPERLU_DIR}/include --with-dsuperlu-lib=\"${HYPRE_SUPERLU_LIBS}\"" ) list( APPEND HYPRE_DEPENDS superlu_dist ) - endif() + list( APPEND HYPRE_CMAKE_ARGS "-DHYPRE_ENABLE_DSUPERLU:BOOL=ON" + "-DTPL_DSUPERLU_INCLUDE_DIRS:STRING=${SUPERLU_DIR}/include" ) - if( ENABLE_OPENMP ) - set( HYPRE_OPENMP_FLAG --with-openmp ) - set( HYPRE_C_FLAGS "${HYPRE_C_FLAGS} ${OpenMP_C_FLAGS}" ) - set( HYPRE_CXX_FLAGS "${HYPRE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" ) + if( BUILD_SHARED_LIBS ) + set( HYPRE_SUPERLU_LIBRARY_SUFFIX so ) + else() + set( HYPRE_SUPERLU_LIBRARY_SUFFIX a ) + endif() + set( HYPRE_TPL_DSUPERLU_LIBRARIES + "${SUPERLU_DIR}/lib/libsuperlu_dist.${HYPRE_SUPERLU_LIBRARY_SUFFIX}" + "${PARMETIS_DIR}/lib/libparmetis.${HYPRE_SUPERLU_LIBRARY_SUFFIX}" + "${METIS_DIR}/lib/libmetis.${HYPRE_SUPERLU_LIBRARY_SUFFIX}" ) + string( REPLACE ";" "^" HYPRE_TPL_DSUPERLU_LIBRARIES_ARG + "${HYPRE_TPL_DSUPERLU_LIBRARIES}" ) + list( APPEND HYPRE_CMAKE_ARGS + "-DTPL_DSUPERLU_LIBRARIES:STRING=${HYPRE_TPL_DSUPERLU_LIBRARIES_ARG}" ) endif() if( ENABLE_HYPRE_DEVICE STREQUAL "CUDA" ) if( NOT ENABLE_CUDA ) message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to CUDA when ENABLE_CUDA is OFF" ) - else() - list( APPEND HYPRE_DEPENDS chai ) - string( SUBSTRING ${CUDA_ARCH} 3 -1 HYPRE_CUDA_SM ) # sm_XY -> XY - set( HYPRE_CUDA_FLAGS "\ ---with-cuda --with-cuda-home=${CUDA_TOOLKIT_ROOT_DIR} \ ---enable-cusparse \ ---enable-cusolver \ ---enable-unified-memory \ ---with-gpu-arch=${HYPRE_CUDA_SM} \ ---with-umpire \ ---with-umpire-include=${CHAI_DIR}/include \ ---with-umpire-lib-dirs=${CHAI_DIR}/lib \ ---with-umpire-libs=umpire " ) - - if( NOT "${CMAKE_CUDA_FLAGS}" STREQUAL "" ) - set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-extra-CUFLAGS=\"${CMAKE_CUDA_FLAGS}\"" ) - endif() - - if( ENABLE_MPI ) - if( MPI_C_INCLUDE_DIRS ) - list( GET MPI_C_INCLUDE_DIRS 0 HYPRE_MPI_INCLUDE_DIR ) - elseif( MPI_C_COMPILER ) - get_filename_component( HYPRE_MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) - get_filename_component( HYPRE_MPI_PREFIX_DIR ${HYPRE_MPI_BIN_DIR} DIRECTORY ) - set( HYPRE_MPI_INCLUDE_DIR "${HYPRE_MPI_PREFIX_DIR}/include" ) - endif() - if( HYPRE_MPI_INCLUDE_DIR ) - set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-MPI-include=${HYPRE_MPI_INCLUDE_DIR}" ) - endif() - endif() - - if( ENABLE_HYPRE_GPU_AWARE_MPI ) - set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) - endif() + endif() - if( ENABLE_HYPRE_GPU_PROFILING ) - set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) - endif() + list( APPEND HYPRE_DEPENDS chai ) + if( NOT DEFINED CMAKE_CUDA_HOST_COMPILER OR CMAKE_CUDA_HOST_COMPILER STREQUAL "" ) + set( HYPRE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" ) + else() + set( HYPRE_CUDA_HOST_COMPILER "${CMAKE_CUDA_HOST_COMPILER}" ) endif() + list( APPEND HYPRE_CMAKE_ARGS + "-DHYPRE_ENABLE_CUDA:BOOL=ON" + "-DHYPRE_ENABLE_UNIFIED_MEMORY:BOOL=ON" + "-DHYPRE_ENABLE_UMPIRE:BOOL=ON" + "-DHYPRE_ENABLE_UMPIRE_DEVICE:BOOL=ON" + "-DHYPRE_ENABLE_UMPIRE_UM:BOOL=ON" + "-DHYPRE_ENABLE_CUDA_STREAMS:BOOL=ON" + "-DHYPRE_ENABLE_CUSPARSE:BOOL=ON" + "-DHYPRE_ENABLE_CUSOLVER:BOOL=ON" + "-DHYPRE_ENABLE_CUBLAS:BOOL=ON" + "-DHYPRE_ENABLE_CURAND:BOOL=ON" + "-DCMAKE_CUDA_COMPILER:FILEPATH=${CMAKE_CUDA_COMPILER}" + "-DCMAKE_CUDA_HOST_COMPILER:FILEPATH=${HYPRE_CUDA_HOST_COMPILER}" + "-DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES}" + "-DCMAKE_CUDA_STANDARD:STRING=${CMAKE_CUDA_STANDARD}" + "-DCMAKE_CUDA_STANDARD_REQUIRED:BOOL=ON" + "-DCMAKE_CUDA_FLAGS:STRING=${CMAKE_CUDA_FLAGS}" + "-DCUDAToolkit_ROOT:PATH=${CUDA_TOOLKIT_ROOT_DIR}" + "-DCUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR}" ) elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) if( NOT ENABLE_HIP ) message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + endif() + + list( APPEND HYPRE_DEPENDS chai ) + if( NOT DEFINED CMAKE_HIP_STANDARD OR CMAKE_HIP_STANDARD STREQUAL "" ) + set( HYPRE_HIP_STANDARD "${CMAKE_CXX_STANDARD}" ) + else() + set( HYPRE_HIP_STANDARD "${CMAKE_HIP_STANDARD}" ) + endif() + list( APPEND HYPRE_CMAKE_ARGS + "-DHYPRE_ENABLE_HIP:BOOL=ON" + "-DHYPRE_ENABLE_UMPIRE:BOOL=ON" + "-DHYPRE_ENABLE_UMPIRE_DEVICE:BOOL=ON" + "-DHYPRE_ENABLE_UMPIRE_UM:BOOL=OFF" + "-DHYPRE_ENABLE_ROCBLAS:BOOL=ON" + "-DHYPRE_ENABLE_ROCSPARSE:BOOL=ON" + "-DHYPRE_ENABLE_ROCSOLVER:BOOL=ON" + "-DHYPRE_ENABLE_ROCRAND:BOOL=ON" + "-DCMAKE_HIP_COMPILER:FILEPATH=${CMAKE_HIP_COMPILER}" + "-DCMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES}" + "-DCMAKE_HIP_STANDARD:STRING=${HYPRE_HIP_STANDARD}" + "-DCMAKE_HIP_STANDARD_REQUIRED:BOOL=ON" + "-DROCM_PATH:PATH=${ROCM_PATH}" ) + endif() + + if( ENABLE_HYPRE_DEVICE STREQUAL "CUDA" OR ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + list( APPEND HYPRE_CMAKE_ARGS "-DHYPRE_ENABLE_GPU_AWARE_MPI:BOOL=ON" ) + endif() + if( ENABLE_HYPRE_GPU_PROFILING ) + list( APPEND HYPRE_CMAKE_ARGS "-DHYPRE_ENABLE_GPU_PROFILING:BOOL=ON" ) + endif() + + if( BUILD_SHARED_LIBS ) + set( HYPRE_UMPIRE_LIBRARY_SUFFIX so ) else() - list( APPEND HYPRE_DEPENDS chai ) - set( HYPRE_HIP_FLAGS "\ -ROCM_PATH=${ROCM_PATH} \ ---with-hip \ ---enable-rocblas \ ---enable-rocsparse \ ---enable-rocsolver \ ---with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ ---with-umpire-device \ ---with-umpire-include=${CHAI_DIR}/include \ ---with-umpire-lib-dirs=${CHAI_DIR}/lib \ ---with-umpire-libs=\"umpire camp\" \ ---with-MPI-include=${MPI_INCLUDE_DIR} " ) - - if( ENABLE_HYPRE_GPU_AWARE_MPI ) - set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) - endif() - - if( ENABLE_HYPRE_GPU_PROFILING ) - set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) - endif() + set( HYPRE_UMPIRE_LIBRARY_SUFFIX a ) endif() + set( HYPRE_TPL_UMPIRE_INCLUDE_DIRS "${CHAI_DIR}/include" ) + set( HYPRE_TPL_UMPIRE_LIBRARIES + "${CHAI_DIR}/lib/libumpire.${HYPRE_UMPIRE_LIBRARY_SUFFIX}" + "${CHAI_DIR}/lib/libcamp.${HYPRE_UMPIRE_LIBRARY_SUFFIX}" + "${CHAI_DIR}/lib/libfmt.${HYPRE_UMPIRE_LIBRARY_SUFFIX}" ) + string( REPLACE ";" "^" HYPRE_TPL_UMPIRE_LIBRARIES_ARG + "${HYPRE_TPL_UMPIRE_LIBRARIES}" ) + list( APPEND HYPRE_CMAKE_ARGS + "-DTPL_UMPIRE_INCLUDE_DIRS:STRING=${HYPRE_TPL_UMPIRE_INCLUDE_DIRS}" + "-DTPL_UMPIRE_LIBRARIES:STRING=${HYPRE_TPL_UMPIRE_LIBRARIES_ARG}" ) endif() if( CMAKE_HOST_APPLE ) - set( HYPRE_INT_FLAG "--enable-bigint" ) + list( APPEND HYPRE_CMAKE_ARGS "-DHYPRE_ENABLE_BIGINT:BOOL=ON" ) else() - set( HYPRE_INT_FLAG "--enable-mixedint" ) + list( APPEND HYPRE_CMAKE_ARGS "-DHYPRE_ENABLE_MIXEDINT:BOOL=ON" ) endif() + string( REPLACE ";" "^" HYPRE_TPL_BLAS_LIBRARIES_ARG "${BLAS_LIBRARIES}" ) + string( REPLACE ";" "^" HYPRE_TPL_LAPACK_LIBRARIES_ARG "${LAPACK_LIBRARIES}" ) + list( APPEND HYPRE_CMAKE_ARGS + "-DTPL_BLAS_LIBRARIES:STRING=${HYPRE_TPL_BLAS_LIBRARIES_ARG}" + "-DTPL_LAPACK_LIBRARIES:STRING=${HYPRE_TPL_LAPACK_LIBRARIES_ARG}" ) + if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) + list( APPEND HYPRE_CMAKE_ARGS + "-DHYPRE_ENABLE_CALIPER:BOOL=ON" + "-DTPL_CALIPER_INCLUDE_DIRS:STRING=${CALIPER_DIR}/include" + "-DTPL_CALIPER_LIBRARIES:STRING=${CALIPER_DIR}/lib/${CALIPER_LIB}" ) endif() - if( CMAKE_BUILD_TYPE MATCHES Debug ) - set( HYPRE_DEBUG_FLAG "--enable-debug" ) - endif() - - file(WRITE ${PROJECT_BINARY_DIR}/config_hypre_for_geos "\ - ./configure \ - CC=${HYPRE_C_COMPILER} \ - CFLAGS=\"${HYPRE_C_FLAGS}\" \ - CXX=${HYPRE_CXX_COMPILER} \ - CXXFLAGS=\"${HYPRE_CXX_FLAGS}\" \ - FC=${HYPRE_Fortran_COMPILER} \ - FCFLAGS=\"${HYPRE_Fortran_FLAGS}\" \ - --prefix=${HYPRE_DIR} \ - --with-print-errors \ - ${HYPRE_DEBUG_FLAG} \ - ${HYPRE_INT_FLAG} \ - --with-blas-lib=\"${BLAS_LINK_LINE}\" \ - --with-lapack-lib=\"${LAPACK_LINK_LINE}\" \ - ${HYPRE_SUPERLU_FLAGS} \ - ${HYPRE_OPENMP_FLAG} \ - ${HYPRE_CUDA_FLAGS} \ - ${HYPRE_HIP_FLAGS} \ - ${HYPRE_CALIPER_FLAGS}\n") - ExternalProject_Add( hypre URL ${HYPRE_URL} URL_HASH SHA256=${HYPRE_URL_HASH} UPDATE_COMMAND "" # Skip git pull/update since this is a snapshot PREFIX ${PROJECT_BINARY_DIR}/hypre + SOURCE_DIR ${PROJECT_BINARY_DIR}/hypre/src/hypre + BINARY_DIR ${PROJECT_BINARY_DIR}/hypre-build + SOURCE_SUBDIR src INSTALL_DIR ${HYPRE_DIR} DEPENDS ${HYPRE_DEPENDS} - BINARY_DIR ${PROJECT_BINARY_DIR}/hypre/src/hypre/src - CONFIGURE_COMMAND cat ${PROJECT_BINARY_DIR}/config_hypre_for_geos && . ${PROJECT_BINARY_DIR}/config_hypre_for_geos - BUILD_COMMAND make -j ${NUM_PROC} VERBOSE=1 - INSTALL_COMMAND make install + LIST_SEPARATOR "^" + CMAKE_GENERATOR ${TPL_GENERATOR} + CMAKE_ARGS ${HYPRE_CMAKE_ARGS} + BUILD_COMMAND ${TPL_BUILD_COMMAND} + INSTALL_COMMAND ${TPL_INSTALL_COMMAND} ) list( APPEND build_list hypre ) @@ -1098,10 +1159,16 @@ if( ENABLE_HYPREDRV ) set( HYPREDRV_URL_HASH "c33212d5fb7391c81ae22ae801d387f41c2c3f293f48ccd8153845d97ba5150c" ) set( HYPREDRV_C_COMPILER ${CMAKE_C_COMPILER} ) set( HYPREDRV_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" ) + set( HYPREDRV_HYPRE_INCLUDE_DIRS "${HYPRE_DIR}/include" ) set( HYPREDRV_HYPRE_LIBRARY "${HYPRE_DIR}/lib/libHYPRE.a" ) set( HYPREDRV_HYPRE_LIBRARIES "${HYPREDRV_HYPRE_LIBRARY}" ) list( APPEND HYPREDRV_HYPRE_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} ) - string( REPLACE ";" "^" HYPREDRV_HYPRE_LIBRARIES_ARG "${HYPREDRV_HYPRE_LIBRARIES}" ) + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) + list( APPEND HYPREDRV_HYPRE_LIBRARIES + "${SUPERLU_DIR}/lib/libsuperlu_dist.a" + "${PARMETIS_DIR}/lib/libparmetis.a" + "${METIS_DIR}/lib/libmetis.a" ) + endif() set( HYPREDRV_MPI_ARGS ) set( HYPREDRV_ACCELERATOR_ARGS ) set( HYPREDRV_ENABLE_CUDA OFF ) @@ -1116,12 +1183,29 @@ if( ENABLE_HYPREDRV ) if( ENABLE_HYPRE_DEVICE STREQUAL "CUDA" ) set( HYPREDRV_ENABLE_CUDA ON ) + # The legacy HYPRE build uses Umpire for device memory. Its generated + # headers include Umpire headers, and its static archive leaves the + # Umpire/CUDA dependencies to consumers, so propagate both explicitly. + list( APPEND HYPREDRV_HYPRE_INCLUDE_DIRS "${CHAI_DIR}/include" ) + list( APPEND HYPREDRV_HYPRE_LIBRARIES + "${CHAI_DIR}/lib/libumpire.a" + "${CHAI_DIR}/lib/libcamp.a" + "${CHAI_DIR}/lib/libfmt.a" + "-L${CUDA_TOOLKIT_ROOT_DIR}/lib64" + -lcudart -lcusparse -lcublas -lcurand -lcusolver + -ldl -lstdc++ ) list( APPEND HYPREDRV_ACCELERATOR_ARGS -D CMAKE_CUDA_COMPILER:FILEPATH=${CMAKE_CUDA_COMPILER} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} ) elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) set( HYPREDRV_ENABLE_HIP ON ) + list( APPEND HYPREDRV_HYPRE_INCLUDE_DIRS "${CHAI_DIR}/include" ) + list( APPEND HYPREDRV_HYPRE_LIBRARIES + "${CHAI_DIR}/lib/libumpire.a" + "${CHAI_DIR}/lib/libcamp.a" + "${CHAI_DIR}/lib/libfmt.a" + -ldl -lstdc++ ) list( APPEND HYPREDRV_ACCELERATOR_ARGS -D CMAKE_HIP_COMPILER:FILEPATH=${CMAKE_HIP_COMPILER} -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} @@ -1139,9 +1223,13 @@ if( ENABLE_HYPREDRV ) message( STATUS "Building HYPREDRIVE found at ${HYPREDRV_URL}" ) - # The legacy superbuild installs HYPRE with its Autotools interface. Pass - # those include/library paths explicitly so HypreDrive does not fetch and - # build a second copy of HYPRE. + string( REPLACE ";" "^" HYPREDRV_HYPRE_INCLUDE_DIRS_ARG + "${HYPREDRV_HYPRE_INCLUDE_DIRS}" ) + string( REPLACE ";" "^" HYPREDRV_HYPRE_LIBRARIES_ARG + "${HYPREDRV_HYPRE_LIBRARIES}" ) + + # Pass the superbuild's HYPRE include/library paths explicitly so + # HypreDrive does not fetch and build a second copy of HYPRE. ExternalProject_Add( hypredrive URL ${HYPREDRV_URL} URL_HASH SHA256=${HYPREDRV_URL_HASH} @@ -1161,7 +1249,7 @@ if( ENABLE_HYPREDRV ) -D CMAKE_INSTALL_LIBDIR:STRING=lib -D BUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=${CMAKE_POSITION_INDEPENDENT_CODE} - -D HYPRE_INCLUDE_DIRS:PATH=${HYPRE_DIR}/include + -D HYPRE_INCLUDE_DIRS:PATH=${HYPREDRV_HYPRE_INCLUDE_DIRS_ARG} -D HYPRE_LIBRARIES:STRING=${HYPREDRV_HYPRE_LIBRARIES_ARG} -D HYPREDRV_ENABLE_TESTING:BOOL=OFF -D HYPREDRV_ENABLE_ALL_TESTS:BOOL=OFF @@ -1447,6 +1535,9 @@ endif( ENABLE_MPI ) -D CMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -D CMAKE_CXX_COMPILER=${VTK_CXX_COMPILER} -D CMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} + -D CMAKE_CXX_STANDARD:STRING=${CXX_STANDARD} + -D CMAKE_CXX_STANDARD_REQUIRED:BOOL=ON + -D VTK_IGNORE_CMAKE_CXX17_CHECKS:BOOL=ON -D VTK_USE_MPI=${VTK_USE_MPI} -D VTK_MODULE_ENABLE_VTK_IOParallelXML=${VTK_MODULE_ENABLE_VTK_IOParallelXML} -D VTK_MODULE_ENABLE_VTK_FiltersParallelDIY2=${VTK_MODULE_ENABLE_VTK_FiltersParallelDIY2} @@ -1466,8 +1557,7 @@ endif( ENABLE_MPI ) -D VTK_MODULE_ENABLE_VTK_vtkm=DONT_WANT -D VTK_MODULE_ENABLE_VTK_IOXML=YES -D VTK_MODULE_ENABLE_VTK_IOLegacy=YES - -D VTK_BUILD_TESTING=OFF - -D VTK_LEGACY_REMOVE=ON ) + -D VTK_BUILD_TESTING=OFF ) list( APPEND build_list vtk ) endif( ENABLE_VTK ) @@ -1508,15 +1598,207 @@ list(APPEND build_list fmt) ################################ message(STATUS "Building = ${build_list}") +# Generate a self-contained configuration for a downstream GEOS build. The +# file is generated during configuration and copied to the install prefix by +# the `tpl` target after all external projects have been built. +set(GEOS_TPL_HOST_CONFIG_TPL_DIR "${CMAKE_INSTALL_PREFIX}") +set(GEOS_TPL_HOST_CONFIG_CAMP_DIR "${CMAKE_INSTALL_PREFIX}/raja") + +# Include the toolchain identity in the host-config name so that an install +# directory makes its compiler/backend requirements visible without opening +# the file. Use the C++ compiler because it is the compiler used by GEOS and +# is also the host compiler for CUDA builds. +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(GEOS_TPL_HOST_CONFIG_COMPILER gcc) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set(GEOS_TPL_HOST_CONFIG_COMPILER appleclang) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(GEOS_TPL_HOST_CONFIG_COMPILER clang) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + set(GEOS_TPL_HOST_CONFIG_COMPILER intelllvm) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set(GEOS_TPL_HOST_CONFIG_COMPILER intel) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + set(GEOS_TPL_HOST_CONFIG_COMPILER nvhpc) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "PGI") + set(GEOS_TPL_HOST_CONFIG_COMPILER pgi) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Cray") + set(GEOS_TPL_HOST_CONFIG_COMPILER cray) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "XL") + set(GEOS_TPL_HOST_CONFIG_COMPILER xl) +else() + string(TOLOWER "${CMAKE_CXX_COMPILER_ID}" GEOS_TPL_HOST_CONFIG_COMPILER) +endif() +if(GEOS_TPL_HOST_CONFIG_COMPILER STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_COMPILER unknown) +endif() + +set(GEOS_TPL_HOST_CONFIG_COMPILER_VERSION "${CMAKE_CXX_COMPILER_VERSION}") +if(GEOS_TPL_HOST_CONFIG_COMPILER_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_COMPILER_VERSION unknown) +endif() +string(REGEX REPLACE "[^A-Za-z0-9._+-]" "_" + GEOS_TPL_HOST_CONFIG_COMPILER_VERSION + "${GEOS_TPL_HOST_CONFIG_COMPILER_VERSION}") + +set(GEOS_TPL_HOST_CONFIG_BACKEND cpu) +set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "") +if(ENABLE_CUDA) + set(GEOS_TPL_HOST_CONFIG_BACKEND cuda) + if(DEFINED CUDAToolkit_VERSION AND NOT CUDAToolkit_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "${CUDAToolkit_VERSION}") + elseif(DEFINED CMAKE_CUDA_COMPILER_VERSION AND NOT CMAKE_CUDA_COMPILER_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "${CMAKE_CUDA_COMPILER_VERSION}") + elseif(DEFINED CUDA_VERSION_STRING AND NOT CUDA_VERSION_STRING STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "${CUDA_VERSION_STRING}") + endif() +elseif(ENABLE_HIP) + set(GEOS_TPL_HOST_CONFIG_BACKEND hip) + if(DEFINED hip_VERSION AND NOT hip_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "${hip_VERSION}") + elseif(DEFINED ROCM_VERSION AND NOT ROCM_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "${ROCM_VERSION}") + elseif(DEFINED CMAKE_HIP_COMPILER_VERSION AND NOT CMAKE_HIP_COMPILER_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION "${CMAKE_HIP_COMPILER_VERSION}") + endif() +endif() +if(NOT GEOS_TPL_HOST_CONFIG_BACKEND STREQUAL "cpu" + AND GEOS_TPL_HOST_CONFIG_BACKEND_VERSION STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_BACKEND_VERSION unknown) +endif() +string(REGEX REPLACE "[^A-Za-z0-9._+-]" "_" + GEOS_TPL_HOST_CONFIG_BACKEND_VERSION + "${GEOS_TPL_HOST_CONFIG_BACKEND_VERSION}") + +set(GEOS_TPL_HOST_CONFIG_FILENAME + "host-config-${GEOS_TPL_HOST_CONFIG_COMPILER}@${GEOS_TPL_HOST_CONFIG_COMPILER_VERSION}-${GEOS_TPL_HOST_CONFIG_BACKEND}") +if(NOT GEOS_TPL_HOST_CONFIG_BACKEND STREQUAL "cpu") + string(APPEND GEOS_TPL_HOST_CONFIG_FILENAME "@${GEOS_TPL_HOST_CONFIG_BACKEND_VERSION}") +endif() +string(APPEND GEOS_TPL_HOST_CONFIG_FILENAME ".cmake") +message(STATUS "Generated GEOS TPL host config name: ${GEOS_TPL_HOST_CONFIG_FILENAME}") + +if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD STREQUAL "") + set(CMAKE_CXX_STANDARD "${CXX_STANDARD}") +endif() +set(GEOS_TPL_HOST_CONFIG_CXX_STANDARD "${CMAKE_CXX_STANDARD}") + +if(DEFINED MPIEXEC_EXECUTABLE AND NOT MPIEXEC_EXECUTABLE STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_MPIEXEC "${MPIEXEC_EXECUTABLE}") +elseif(DEFINED MPIEXEC AND NOT MPIEXEC STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_MPIEXEC "${MPIEXEC}") +elseif(DEFINED MPI_C_COMPILER AND NOT MPI_C_COMPILER STREQUAL "") + get_filename_component(GEOS_TPL_HOST_CONFIG_MPI_BIN "${MPI_C_COMPILER}" DIRECTORY) + set(GEOS_TPL_HOST_CONFIG_MPIEXEC "${GEOS_TPL_HOST_CONFIG_MPI_BIN}/mpirun") +else() + set(GEOS_TPL_HOST_CONFIG_MPIEXEC "") +endif() +if(DEFINED MPIEXEC_NUMPROC_FLAG AND NOT MPIEXEC_NUMPROC_FLAG STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_MPIEXEC_NUMPROC_FLAG "${MPIEXEC_NUMPROC_FLAG}") +else() + set(GEOS_TPL_HOST_CONFIG_MPIEXEC_NUMPROC_FLAG "-np") +endif() + +if(DEFINED ENABLE_HYPRE_DEVICE AND NOT ENABLE_HYPRE_DEVICE STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_HYPRE_DEVICE "${ENABLE_HYPRE_DEVICE}") +else() + set(GEOS_TPL_HOST_CONFIG_HYPRE_DEVICE CPU) +endif() + +if(DEFINED ENABLE_HYPRE_GPU_AWARE_MPI) + set(GEOS_TPL_HOST_CONFIG_HYPRE_GPU_AWARE_MPI "${ENABLE_HYPRE_GPU_AWARE_MPI}") +else() + set(GEOS_TPL_HOST_CONFIG_HYPRE_GPU_AWARE_MPI OFF) +endif() +if(DEFINED ENABLE_CALIPER_HYPRE) + set(GEOS_TPL_HOST_CONFIG_CALIPER_HYPRE "${ENABLE_CALIPER_HYPRE}") +else() + set(GEOS_TPL_HOST_CONFIG_CALIPER_HYPRE OFF) +endif() + +if(ENABLE_HYPRE) + set(GEOS_TPL_HOST_CONFIG_LA_INTERFACE Hypre) +elseif(ENABLE_TRILINOS) + set(GEOS_TPL_HOST_CONFIG_LA_INTERFACE Trilinos) +elseif(ENABLE_PETSC) + set(GEOS_TPL_HOST_CONFIG_LA_INTERFACE Petsc) +else() + set(GEOS_TPL_HOST_CONFIG_LA_INTERFACE Hypre) +endif() + +if(ENABLE_MPI) + set(GEOS_TPL_HOST_CONFIG_PARMETIS ON) +else() + set(GEOS_TPL_HOST_CONFIG_PARMETIS OFF) +endif() +set(GEOS_TPL_HOST_CONFIG_METIS ON) +set(GEOS_TPL_HOST_CONFIG_SILO ON) +set(GEOS_TPL_HOST_CONFIG_UNCRUSTIFY ON) +if(CMAKE_HOST_APPLE) + set(GEOS_TPL_HOST_CONFIG_HYPRE_MIXINT OFF) +else() + set(GEOS_TPL_HOST_CONFIG_HYPRE_MIXINT ON) +endif() +if(DEFINED MATHPRESSO_DIR AND NOT MATHPRESSO_DIR STREQUAL "") + set(GEOS_TPL_HOST_CONFIG_MATHPRESSO ON) +else() + set(GEOS_TPL_HOST_CONFIG_MATHPRESSO OFF) +endif() +set(GEOS_TPL_HOST_CONFIG_DOXYGEN_EXECUTABLE "${CMAKE_INSTALL_PREFIX}/doxygen/bin/doxygen") +set(GEOS_TPL_HOST_CONFIG_UNCRUSTIFY_EXECUTABLE "${CMAKE_INSTALL_PREFIX}/uncrustify/bin/uncrustify") + +# CUDA 13 no longer ships nvToolsExt. Do not propagate a stale superbuild +# option to GEOS when the library is absent from the selected toolkit. +set(GEOS_TPL_HOST_CONFIG_CUDA_NVTOOLSEXT OFF) +if(ENABLE_CUDA) + find_library(GEOS_TPL_NVTOOLSEXT_LIBRARY + NAMES nvToolsExt + PATHS "${CUDA_TOOLKIT_ROOT_DIR}/lib64" "${CUDA_TOOLKIT_ROOT_DIR}/lib" + NO_DEFAULT_PATH) + if(GEOS_TPL_NVTOOLSEXT_LIBRARY) + set(GEOS_TPL_HOST_CONFIG_CUDA_NVTOOLSEXT ON) + elseif(ENABLE_CUDA_NVTOOLSEXT) + message(WARNING "ENABLE_CUDA_NVTOOLSEXT is ON, but nvToolsExt was not found; disabling it in the generated GEOS host config") + endif() + unset(GEOS_TPL_NVTOOLSEXT_LIBRARY CACHE) +endif() + +set(GEOS_TPL_HOST_CONFIG_PREFIX_PATH + "${CMAKE_INSTALL_PREFIX}/adiak;${CMAKE_INSTALL_PREFIX}/caliper;${CMAKE_INSTALL_PREFIX}/chai;${CMAKE_INSTALL_PREFIX}/conduit;${CMAKE_INSTALL_PREFIX}/fmt;${CMAKE_INSTALL_PREFIX}/hdf5;${CMAKE_INSTALL_PREFIX}/hypre;${CMAKE_INSTALL_PREFIX}/hypredrive;${CMAKE_INSTALL_PREFIX}/mathpresso;${CMAKE_INSTALL_PREFIX}/metis;${CMAKE_INSTALL_PREFIX}/parmetis;${CMAKE_INSTALL_PREFIX}/pugixml;${CMAKE_INSTALL_PREFIX}/raja;${CMAKE_INSTALL_PREFIX}/scotch;${CMAKE_INSTALL_PREFIX}/silo;${CMAKE_INSTALL_PREFIX}/suitesparse;${CMAKE_INSTALL_PREFIX}/superlu_dist;${CMAKE_INSTALL_PREFIX}/vtk") + +set(GEOS_TPL_HOST_CONFIG_BUILD "${PROJECT_BINARY_DIR}/${GEOS_TPL_HOST_CONFIG_FILENAME}") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/geos-host-config.cmake.in" + "${GEOS_TPL_HOST_CONFIG_BUILD}" @ONLY) +install(FILES "${GEOS_TPL_HOST_CONFIG_BUILD}" DESTINATION .) + blt_add_executable( NAME tpl SOURCES tpl.cpp ) add_dependencies( tpl ${build_list} ) +# Keep the installed copy current even when the `tpl` executable itself is +# already up to date after a CMake reconfigure. +set(GEOS_TPL_HOST_CONFIG_INSTALL "${CMAKE_INSTALL_PREFIX}/${GEOS_TPL_HOST_CONFIG_FILENAME}") +add_custom_command( + OUTPUT "${GEOS_TPL_HOST_CONFIG_INSTALL}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_INSTALL_PREFIX}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${GEOS_TPL_HOST_CONFIG_BUILD}" + "${GEOS_TPL_HOST_CONFIG_INSTALL}" + DEPENDS "${GEOS_TPL_HOST_CONFIG_BUILD}" + COMMENT "Installing generated GEOS host config") +add_custom_target( geos_tpl_host_config ALL + DEPENDS "${GEOS_TPL_HOST_CONFIG_INSTALL}" ) +add_dependencies( geos_tpl_host_config tpl ) + # Clean up add_custom_command( TARGET tpl POST_BUILD COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/config_silo_for_geos - COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/config_hypre_for_geos COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/config_suitesparse_for_geos COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/build_suitesparse_for_geos - COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/install_suitesparse_for_geos ) + COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/install_suitesparse_for_geos + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX} + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${GEOS_TPL_HOST_CONFIG_BUILD} + ${GEOS_TPL_HOST_CONFIG_INSTALL} + COMMENT "Installing generated GEOS host config") diff --git a/cmake/geos-host-config.cmake.in b/cmake/geos-host-config.cmake.in new file mode 100644 index 00000000..5f47ac37 --- /dev/null +++ b/cmake/geos-host-config.cmake.in @@ -0,0 +1,116 @@ +# Generated by the legacy GEOS TPL build. +# Configuration: @GEOS_TPL_HOST_CONFIG_FILENAME@ +# +# This file configures a downstream GEOS build. It intentionally does not +# set CMAKE_INSTALL_PREFIX, so the GEOS installation location remains a +# downstream choice. + +set(GEOS_TPL_HOST_CONFIG_LOADED ON CACHE INTERNAL "Generated GEOS TPL host config loaded") +set(GEOS_TPL_DIR "@GEOS_TPL_HOST_CONFIG_TPL_DIR@" CACHE PATH "GEOS third-party library installation") + +set(CMAKE_BUILD_TYPE "@CMAKE_BUILD_TYPE@" CACHE STRING "") +set(CMAKE_C_COMPILER "@CMAKE_C_COMPILER@" CACHE FILEPATH "") +set(CMAKE_CXX_COMPILER "@CMAKE_CXX_COMPILER@" CACHE FILEPATH "") +set(CMAKE_Fortran_COMPILER "@CMAKE_Fortran_COMPILER@" CACHE FILEPATH "") +set(CMAKE_CXX_STANDARD "@GEOS_TPL_HOST_CONFIG_CXX_STANDARD@" CACHE STRING "") +set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL "") +set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "") +set(BLT_CXX_STD "@BLT_CXX_STD@" CACHE STRING "") +set(ENABLE_FORTRAN OFF CACHE BOOL "") +set(CMAKE_C_FLAGS "@CMAKE_C_FLAGS@" CACHE STRING "") +set(CMAKE_CXX_FLAGS "@CMAKE_CXX_FLAGS@" CACHE STRING "") + +set(ENABLE_MPI "@ENABLE_MPI@" CACHE BOOL "") +set(MPI_C_COMPILER "@MPI_C_COMPILER@" CACHE FILEPATH "") +set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@" CACHE FILEPATH "") +set(MPI_Fortran_COMPILER "@MPI_Fortran_COMPILER@" CACHE FILEPATH "") +set(MPIEXEC_EXECUTABLE "@GEOS_TPL_HOST_CONFIG_MPIEXEC@" CACHE FILEPATH "") +set(MPIEXEC_NUMPROC_FLAG "@GEOS_TPL_HOST_CONFIG_MPIEXEC_NUMPROC_FLAG@" CACHE STRING "") + +set(CMAKE_PREFIX_PATH "@GEOS_TPL_HOST_CONFIG_PREFIX_PATH@" CACHE STRING "") +set(RAJA_DIR "@RAJA_DIR@" CACHE PATH "") +set(CAMP_DIR "@GEOS_TPL_HOST_CONFIG_CAMP_DIR@" CACHE PATH "") +set(UMPIRE_DIR "@CHAI_DIR@" CACHE PATH "") +set(CHAI_DIR "@CHAI_DIR@" CACHE PATH "") +set(HDF5_DIR "@HDF5_DIR@" CACHE PATH "") +set(CONDUIT_DIR "@CONDUIT_DIR@" CACHE PATH "") +set(SILO_DIR "@SILO_DIR@" CACHE PATH "") +set(ADIAK_DIR "@ADIAK_DIR@" CACHE PATH "") +set(CALIPER_DIR "@CALIPER_DIR@" CACHE PATH "") +set(PUGIXML_DIR "@PUGIXML_DIR@" CACHE PATH "") +set(VTK_DIR "@VTK_DIR@" CACHE PATH "") +set(FMT_DIR "@FMT_DIR@" CACHE PATH "") +set(METIS_DIR "@METIS_DIR@" CACHE PATH "") +set(PARMETIS_DIR "@PARMETIS_DIR@" CACHE PATH "") +set(SUPERLU_DIST_DIR "@SUPERLU_DIR@" CACHE PATH "") +set(SUITESPARSE_DIR "@SUITESPARSE_DIR@" CACHE PATH "") +# HypreDrive's package config may probe for a CMake HYPRE package while GEOS +# consumes the superbuild's headers and archive directly. That probe can leave +# HYPRE_DIR as HYPRE_DIR-NOTFOUND; recover that value on the next configure +# while preserving a valid explicit -DHYPRE_DIR override. +if(NOT DEFINED HYPRE_DIR OR HYPRE_DIR MATCHES "-NOTFOUND$") + set(HYPRE_DIR "@HYPRE_DIR@" CACHE PATH "" FORCE) +endif() +set(HYPREDRV_DIR "@HYPREDRV_DIR@" CACHE PATH "") +set(SCOTCH_DIR "@SCOTCH_DIR@" CACHE PATH "") +set(PETSC_DIR "@PETSC_INSTALL_DIR@" CACHE PATH "") +set(MATHPRESSO_DIR "@MATHPRESSO_DIR@" CACHE PATH "") +set(DOXYGEN_EXECUTABLE "@GEOS_TPL_HOST_CONFIG_DOXYGEN_EXECUTABLE@" CACHE FILEPATH "") +set(UNCRUSTIFY_EXECUTABLE "@GEOS_TPL_HOST_CONFIG_UNCRUSTIFY_EXECUTABLE@" CACHE FILEPATH "") + +set(ENABLE_CALIPER "@ENABLE_CALIPER@" CACHE BOOL "") +set(ENABLE_CUDA "@ENABLE_CUDA@" CACHE BOOL "") +set(ENABLE_CUDA_NVTOOLSEXT "@GEOS_TPL_HOST_CONFIG_CUDA_NVTOOLSEXT@" CACHE BOOL "") +set(ENABLE_HIP "@ENABLE_HIP@" CACHE BOOL "") +set(ENABLE_OPENMP "@ENABLE_OPENMP@" CACHE BOOL "") +set(ENABLE_HYPRE "@ENABLE_HYPRE@" CACHE BOOL "") +set(ENABLE_HYPREDRV "@ENABLE_HYPREDRV@" CACHE BOOL "") +set(ENABLE_HYPRE_DEVICE "@GEOS_TPL_HOST_CONFIG_HYPRE_DEVICE@" CACHE STRING "") +set(ENABLE_HYPRE_GPU_AWARE_MPI "@GEOS_TPL_HOST_CONFIG_HYPRE_GPU_AWARE_MPI@" CACHE BOOL "") +set(ENABLE_HYPRE_GPU_PROFILING "@ENABLE_HYPRE_GPU_PROFILING@" CACHE BOOL "") +set(ENABLE_CALIPER_HYPRE "@GEOS_TPL_HOST_CONFIG_CALIPER_HYPRE@" CACHE BOOL "") +set(ENABLE_HYPRE_MIXINT "@GEOS_TPL_HOST_CONFIG_HYPRE_MIXINT@" CACHE BOOL "") +set(GEOS_LA_INTERFACE "@GEOS_TPL_HOST_CONFIG_LA_INTERFACE@" CACHE STRING "") +set(ENABLE_TRILINOS "@ENABLE_TRILINOS@" CACHE BOOL "") +set(ENABLE_PETSC "@ENABLE_PETSC@" CACHE BOOL "") +set(ENABLE_METIS "@GEOS_TPL_HOST_CONFIG_METIS@" CACHE BOOL "") +set(ENABLE_PARMETIS "@GEOS_TPL_HOST_CONFIG_PARMETIS@" CACHE BOOL "") +set(ENABLE_SCOTCH "@ENABLE_SCOTCH@" CACHE BOOL "") +set(ENABLE_SUPERLU_DIST "@ENABLE_SUPERLU_DIST@" CACHE BOOL "") +set(ENABLE_SUITESPARSE "@ENABLE_SUITESPARSE@" CACHE BOOL "") +set(ENABLE_SILO "@GEOS_TPL_HOST_CONFIG_SILO@" CACHE BOOL "") +set(ENABLE_MATHPRESSO "@GEOS_TPL_HOST_CONFIG_MATHPRESSO@" CACHE BOOL "") +set(ENABLE_VTK "@ENABLE_VTK@" CACHE BOOL "") +set(ENABLE_VTK_WRAP_PYTHON "@ENABLE_VTK_WRAP_PYTHON@" CACHE BOOL "") +set(ENABLE_UNCRUSTIFY "@GEOS_TPL_HOST_CONFIG_UNCRUSTIFY@" CACHE BOOL "") +set(ENABLE_DOXYGEN OFF CACHE BOOL "") +set(ENABLE_DOCS OFF CACHE BOOL "") +set(ENABLE_EXAMPLES OFF CACHE BOOL "") +set(ENABLE_BENCHMARKS OFF CACHE BOOL "") +set(ENABLE_TESTS ON CACHE BOOL "") +set(GEOS_ENABLE_TESTS ON CACHE BOOL "") + +if("@ENABLE_CUDA@") + set(CMAKE_CUDA_COMPILER "@CMAKE_CUDA_COMPILER@" CACHE FILEPATH "") + set(CMAKE_CUDA_HOST_COMPILER "@CMAKE_CUDA_HOST_COMPILER@" CACHE FILEPATH "") + set(CUDA_TOOLKIT_ROOT_DIR "@CUDA_TOOLKIT_ROOT_DIR@" CACHE PATH "") + set(CUDA_ARCH "@CUDA_ARCH@" CACHE STRING "") + set(CMAKE_CUDA_ARCHITECTURES "@CMAKE_CUDA_ARCHITECTURES@" CACHE STRING "") + set(CMAKE_CUDA_STANDARD "@CMAKE_CUDA_STANDARD@" CACHE STRING "") + set(CMAKE_CUDA_STANDARD_REQUIRED ON CACHE BOOL "") + set(CMAKE_CUDA_FLAGS "@CMAKE_CUDA_FLAGS@" CACHE STRING "") +endif() + +if("@ENABLE_HIP@") + set(CMAKE_HIP_COMPILER "@CMAKE_HIP_COMPILER@" CACHE FILEPATH "") + set(CMAKE_HIP_ARCHITECTURES "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "") + set(CMAKE_HIP_STANDARD "@CMAKE_HIP_STANDARD@" CACHE STRING "") + set(CMAKE_HIP_STANDARD_REQUIRED ON CACHE BOOL "") + set(ROCM_PATH "@ROCM_PATH@" CACHE PATH "") + set(HIP_ROOT "@HIP_ROOT@" CACHE PATH "") + set(GPU_TARGETS "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "") + set(AMDGPU_TARGETS "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "") +endif() + +set(BLAS_LIBRARIES "@BLAS_LIBRARIES@" CACHE STRING "") +set(LAPACK_LIBRARIES "@LAPACK_LIBRARIES@" CACHE STRING "") From 33bea15464bff3d6bab9a8fcad76b460f44c087a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 1 Sep 2026 00:16:57 -0400 Subject: [PATCH 20/22] GPU build fixes --- CMakeLists.txt | 26 ++++++++++++++++---------- cmake/geos-host-config.cmake.in | 6 ++++-- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d825cf1..011a1d00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,18 +21,24 @@ set( ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE ) set( CXX_STANDARD 20 ) set( BLT_CXX_STD c++20 CACHE STRING "" FORCE ) -# CUDA 13.3 rejects newer host compiler versions unless this opt-in is given. -# Keep it in the common CUDA flags so CMake/BLT and the external CUDA TPLs use -# the same NVCC behavior. +# Keep required NVCC options in the common CUDA flags so CMake/BLT, the +# generated GEOS host config, and the external CUDA TPLs use the same behavior. +# CUDA 13.3 rejects newer host compiler versions unless the first option is +# given, and GEOS/LvArray device lambdas require the second one. if( ENABLE_CUDA ) - string( FIND "${CMAKE_CUDA_FLAGS}" "--allow-unsupported-compiler" _GEOS_TPL_CUDA_FLAG_POS ) - if( _GEOS_TPL_CUDA_FLAG_POS EQUAL -1 ) - if( CMAKE_CUDA_FLAGS ) - set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --allow-unsupported-compiler" CACHE STRING "" FORCE ) - else() - set( CMAKE_CUDA_FLAGS "--allow-unsupported-compiler" CACHE STRING "" FORCE ) + foreach( _GEOS_TPL_CUDA_REQUIRED_FLAG IN ITEMS + --allow-unsupported-compiler + --extended-lambda ) + string( FIND "${CMAKE_CUDA_FLAGS}" "${_GEOS_TPL_CUDA_REQUIRED_FLAG}" _GEOS_TPL_CUDA_FLAG_POS ) + if( _GEOS_TPL_CUDA_FLAG_POS EQUAL -1 ) + if( CMAKE_CUDA_FLAGS ) + set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${_GEOS_TPL_CUDA_REQUIRED_FLAG}" CACHE STRING "" FORCE ) + else() + set( CMAKE_CUDA_FLAGS "${_GEOS_TPL_CUDA_REQUIRED_FLAG}" CACHE STRING "" FORCE ) + endif() endif() - endif() + endforeach() + unset( _GEOS_TPL_CUDA_REQUIRED_FLAG ) unset( _GEOS_TPL_CUDA_FLAG_POS ) endif() diff --git a/cmake/geos-host-config.cmake.in b/cmake/geos-host-config.cmake.in index 5f47ac37..26e36ed3 100644 --- a/cmake/geos-host-config.cmake.in +++ b/cmake/geos-host-config.cmake.in @@ -90,7 +90,9 @@ set(ENABLE_BENCHMARKS OFF CACHE BOOL "") set(ENABLE_TESTS ON CACHE BOOL "") set(GEOS_ENABLE_TESTS ON CACHE BOOL "") -if("@ENABLE_CUDA@") +# Compare the substituted option as a string because this cache file is +# loaded before the downstream project establishes CMake's boolean policies. +if("@ENABLE_CUDA@" STREQUAL "ON") set(CMAKE_CUDA_COMPILER "@CMAKE_CUDA_COMPILER@" CACHE FILEPATH "") set(CMAKE_CUDA_HOST_COMPILER "@CMAKE_CUDA_HOST_COMPILER@" CACHE FILEPATH "") set(CUDA_TOOLKIT_ROOT_DIR "@CUDA_TOOLKIT_ROOT_DIR@" CACHE PATH "") @@ -101,7 +103,7 @@ if("@ENABLE_CUDA@") set(CMAKE_CUDA_FLAGS "@CMAKE_CUDA_FLAGS@" CACHE STRING "") endif() -if("@ENABLE_HIP@") +if("@ENABLE_HIP@" STREQUAL "ON") set(CMAKE_HIP_COMPILER "@CMAKE_HIP_COMPILER@" CACHE FILEPATH "") set(CMAKE_HIP_ARCHITECTURES "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "") set(CMAKE_HIP_STANDARD "@CMAKE_HIP_STANDARD@" CACHE STRING "") From ab85fff7c908eb4771625fd4d53a0f86e8e8c37b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 1 Sep 2026 09:16:52 -0400 Subject: [PATCH 21/22] HIP build fixes --- CMakeLists.txt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 011a1d00..5f87dbf6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.23.1) if( POLICY CMP0135 ) cmake_policy( SET CMP0135 NEW ) endif() +if( POLICY CMP0144 ) + cmake_policy( SET CMP0144 NEW ) +endif() ################################ # GEOS_TPL ################################ @@ -239,8 +242,9 @@ if( GEOSXTPL_ENABLE_DOXYGEN ) URL_HASH SHA256=${DOXYGEN_URL_HASH} PREFIX ${PROJECT_BINARY_DIR}/doxygen INSTALL_DIR ${DOXYGEN_DIR} - BUILD_COMMAND make -j ${NUM_PROC} - INSTALL_COMMAND make install + CMAKE_GENERATOR ${TPL_GENERATOR} + BUILD_COMMAND ${CMAKE_COMMAND} --build --parallel ${NUM_PROC} + INSTALL_COMMAND ${CMAKE_COMMAND} --build --target install --parallel ${NUM_PROC} CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_INSTALL_PREFIX= @@ -1006,7 +1010,8 @@ if( ENABLE_HYPRE ) # uses CMAKE_CXX_COMPILER as nvcc's host compiler. list( APPEND HYPRE_CMAKE_ARGS "-DMPI_C_COMPILER:FILEPATH=${MPI_C_COMPILER}" - "-DMPI_CXX_COMPILER:FILEPATH=${MPI_CXX_COMPILER}" ) + "-DMPI_CXX_COMPILER:FILEPATH=${MPI_CXX_COMPILER}" + "-DMPI_INCLUDE_DIR:PATH=${MPI_INCLUDE_DIR}" ) endif() if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) @@ -1211,6 +1216,8 @@ if( ENABLE_HYPREDRV ) "${CHAI_DIR}/lib/libumpire.a" "${CHAI_DIR}/lib/libcamp.a" "${CHAI_DIR}/lib/libfmt.a" + "-L${ROCM_PATH}/lib" + -lrocblas -lrocsparse -lrocrand -lrocsolver -lamdhip64 -ldl -lstdc++ ) list( APPEND HYPREDRV_ACCELERATOR_ARGS -D CMAKE_HIP_COMPILER:FILEPATH=${CMAKE_HIP_COMPILER} From 4d3b9d74a4b6b6e5586c480ba9f6458d9a7fb413 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 3 Sep 2026 01:49:15 -0400 Subject: [PATCH 22/22] Keep TPL fixes on CUDA branch --- CMakeLists.txt | 129 +++++++++++++++--- cmake/apply-patches.cmake | 52 +++++++ cmake/geos-host-config.cmake.in | 3 +- patch/hypre-hip-rocsparse-sort.patch | 24 ++++ patch/hypre-hip-standard-ilu.patch | 28 ++++ patch/hypre-mixedint-export.patch | 12 ++ patch/hypre-rcm-capacity.patch | 7 + patch/hypre-tagged-innerprod.patch | 8 ++ patch/hypre-umpire-wrapper-lifetime.patch | 87 ++++++++++++ patch/hypredrive-cxx-linker.patch | 25 ++++ patch/hypredrive-hip-rocsparse.patch | 77 +++++++++++ patch/raja-reducer-assignment.patch | 94 +++++++++++++ .../spack_packages/packages/geosx/package.py | 35 +++-- .../spack_packages/packages/hypre/package.py | 13 +- .../patches/hypre-hip-rocsparse-sort.patch | 24 ++++ .../patches/hypre-hip-standard-ilu.patch | 28 ++++ .../hypre-mgr-col-lumped-destroy.patch | 13 ++ .../hypre/patches/hypre-mixedint-export.patch | 12 ++ .../hypre/patches/hypre-rcm-capacity.patch | 7 + .../patches/hypre-tagged-innerprod.patch | 8 ++ .../hypre-umpire-wrapper-lifetime.patch | 87 ++++++++++++ .../packages/hypredrive/package.py | 31 +++-- .../patches/hypredrive-cxx-linker.patch | 25 ++++ .../patches/hypredrive-hip-rocsparse.patch | 77 +++++++++++ .../packages/parmetis/package.py | 5 + .../parmetis/patches/parmetis-idx64.patch | 13 ++ .../spack_packages/packages/raja/package.py | 12 ++ .../patches/raja-reducer-assignment.patch | 94 +++++++++++++ 28 files changed, 977 insertions(+), 53 deletions(-) create mode 100644 cmake/apply-patches.cmake create mode 100644 patch/hypre-hip-rocsparse-sort.patch create mode 100644 patch/hypre-hip-standard-ilu.patch create mode 100644 patch/hypre-mixedint-export.patch create mode 100644 patch/hypre-rcm-capacity.patch create mode 100644 patch/hypre-tagged-innerprod.patch create mode 100644 patch/hypre-umpire-wrapper-lifetime.patch create mode 100644 patch/hypredrive-cxx-linker.patch create mode 100644 patch/hypredrive-hip-rocsparse.patch create mode 100644 patch/raja-reducer-assignment.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-hip-rocsparse-sort.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-hip-standard-ilu.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-mgr-col-lumped-destroy.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-mixedint-export.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-rcm-capacity.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-tagged-innerprod.patch create mode 100644 scripts/spack_packages/packages/hypre/patches/hypre-umpire-wrapper-lifetime.patch create mode 100644 scripts/spack_packages/packages/hypredrive/patches/hypredrive-cxx-linker.patch create mode 100644 scripts/spack_packages/packages/hypredrive/patches/hypredrive-hip-rocsparse.patch create mode 100644 scripts/spack_packages/packages/parmetis/patches/parmetis-idx64.patch create mode 100644 scripts/spack_packages/packages/raja/package.py create mode 100644 scripts/spack_packages/packages/raja/patches/raja-reducer-assignment.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f87dbf6..ff364bf3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,6 +86,8 @@ option( ENABLE_VTK_WRAP_PYTHON "" OFF ) option( ENABLE_NINJA "" OFF ) +set( GEOS_TPL_PATCH_SCRIPT "${CMAKE_SOURCE_DIR}/cmake/apply-patches.cmake" ) + include(ProcessorCount) ProcessorCount(NUM_CORES) @@ -279,9 +281,15 @@ endif() if( CMAKE_BUILD_TYPE MATCHES Debug ) set( HDF5_CMAKE_BUILD_TYPE RelWithDebInfo ) else() - set( HDF5_CMAKE_BUILD_TYPE CMAKE_BUILD_TYPE ) + set( HDF5_CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE} ) endif() +set( HDF5_CMAKE_C_FLAGS "${C_FLAGS_NO_WARNINGS}" ) +set( HDF5_CMAKE_CXX_FLAGS "${CXX_FLAGS_NO_WARNINGS}" ) +set( HDF5_CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}" ) +set( HDF5_CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}" ) +set( HDF5_CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS}" ) + ExternalProject_Add( hdf5 URL ${HDF5_URL} URL_HASH SHA256=${HDF5_URL_HASH} @@ -292,10 +300,13 @@ ExternalProject_Add( hdf5 CMAKE_GENERATOR ${TPL_GENERATOR} CMAKE_ARGS -D CMAKE_C_COMPILER:STRING=${HDF5_C_COMPILER} -D CMAKE_CXX_COMPILER:STRING=${HDF5_CXX_COMPILER} - -D CMAKE_C_FLAGS:STRING=${C_FLAGS_NO_WARNINGS} + -D CMAKE_C_FLAGS:STRING=${HDF5_CMAKE_C_FLAGS} -D CMAKE_C_FLAGS_RELEASE:STRING=${CMAKE_C_FLAGS_RELEASE} - -D CMAKE_CXX_FLAGS:STRING=${CXX_FLAGS_NO_WARNINGS} + -D CMAKE_CXX_FLAGS:STRING=${HDF5_CMAKE_CXX_FLAGS} -D CMAKE_CXX_FLAGS_RELEASE:STRING=${CMAKE_CXX_FLAGS_RELEASE} + -D CMAKE_EXE_LINKER_FLAGS:STRING=${HDF5_CMAKE_EXE_LINKER_FLAGS} + -D CMAKE_SHARED_LINKER_FLAGS:STRING=${HDF5_CMAKE_SHARED_LINKER_FLAGS} + -D CMAKE_MODULE_LINKER_FLAGS:STRING=${HDF5_CMAKE_MODULE_LINKER_FLAGS} -D CMAKE_BUILD_TYPE:STRING=${HDF5_CMAKE_BUILD_TYPE} -D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE} -D HDF5_ENABLE_PARALLEL:BOOL=${ENABLE_MPI} @@ -389,6 +400,7 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/config_silo_for_geos "\ CC=\"${HDF5_C_COMPILER}\" \ CXX=\"${HDF5_CXX_COMPILER}\" \ CFLAGS=\"${HDF5_C_FLAGS_STR} ${C_FLAGS_NO_WARNINGS} -Wno-error=implicit-function-declaration -Wno-error=int-conversion\" \ +LDFLAGS=\"${CMAKE_EXE_LINKER_FLAGS}\" \ LIBS=\"-ldl\" \ --prefix=${SILO_DIR} \ --build=${SILO_BUILD_TYPE} \ @@ -450,12 +462,19 @@ if( ENABLE_CUDA ) set( CHAI_CUDA_FLAGS "${CHAI_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS}" ) endif() +set( RAJA_PATCH_COMMAND + ${CMAKE_COMMAND} + "-DGEOS_PATCH_STRIP:STRING=1" + "-DGEOS_PATCH_FILES:STRING=${CMAKE_SOURCE_DIR}/patch/raja-reducer-assignment.patch" + -P ${GEOS_TPL_PATCH_SCRIPT} ) + ExternalProject_Add( raja URL ${RAJA_URL} URL_HASH SHA256=${RAJA_URL_HASH} PREFIX ${PROJECT_BINARY_DIR}/raja INSTALL_DIR ${RAJA_DIR} DEPENDS ${RAJA_DEPENDS} + PATCH_COMMAND ${RAJA_PATCH_COMMAND} BUILD_COMMAND ${TPL_BUILD_COMMAND} INSTALL_COMMAND ${TPL_INSTALL_COMMAND} CMAKE_GENERATOR ${TPL_GENERATOR} @@ -849,7 +868,10 @@ if( ${ENABLE_MPI} ) URL ${PARMETIS_URL} URL_HASH SHA256=${PARMETIS_URL_HASH} INSTALL_DIR ${PARMETIS_DIR} - PATCH_COMMAND patch --forward -p1 < ${CMAKE_SOURCE_DIR}/patch/parmetis-idx64.patch + PATCH_COMMAND ${CMAKE_COMMAND} + "-DGEOS_PATCH_STRIP:STRING=1" + "-DGEOS_PATCH_FILES:STRING=${CMAKE_SOURCE_DIR}/patch/parmetis-idx64.patch" + -P ${GEOS_TPL_PATCH_SCRIPT} BUILD_COMMAND ${TPL_BUILD_COMMAND} INSTALL_COMMAND ${TPL_INSTALL_COMMAND} CMAKE_GENERATOR ${TPL_GENERATOR} @@ -877,7 +899,10 @@ ExternalProject_Add( metis URL_HASH SHA256=${PARMETIS_URL_HASH} INSTALL_DIR ${METIS_DIR} SOURCE_SUBDIR metis - PATCH_COMMAND patch --forward -p1 < ${CMAKE_SOURCE_DIR}/patch/parmetis-idx64.patch + PATCH_COMMAND ${CMAKE_COMMAND} + "-DGEOS_PATCH_STRIP:STRING=1" + "-DGEOS_PATCH_FILES:STRING=${CMAKE_SOURCE_DIR}/patch/parmetis-idx64.patch" + -P ${GEOS_TPL_PATCH_SCRIPT} BUILD_COMMAND ${TPL_BUILD_COMMAND} INSTALL_COMMAND ${TPL_INSTALL_COMMAND} CMAKE_GENERATOR ${TPL_GENERATOR} @@ -964,6 +989,27 @@ if( ENABLE_HYPRE ) set( HYPRE_URL "https://github.com/hypre-space/hypre/archive/53927ee4fb68e5caf6be1957af2c3d75905b6b3d.tar.gz" ) set( HYPRE_URL_HASH "d418846bc1466616cc22e71c4d3b855127846307bb4f4a2f4d37f046ac9c3355" ) set( HYPRE_DEPENDS "" ) + set( HYPRE_PATCH_FILES + "${CMAKE_SOURCE_DIR}/patch/hypre-mixedint-export.patch|" + "${CMAKE_SOURCE_DIR}/patch/hypre-tagged-innerprod.patch|" + "${CMAKE_SOURCE_DIR}/patch/hypre-rcm-capacity.patch|" + "${CMAKE_SOURCE_DIR}/patch/hypre-umpire-wrapper-lifetime.patch" ) + + if( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) + # Avoid the broken legacy rocSPARSE CSR sort/ILU analysis on gfx1100. + # Keep classical HYPRE ILU(0) setup; only the setup-time factor migration + # and triangular application remain device compatible. + string( APPEND HYPRE_PATCH_FILES + "|${CMAKE_SOURCE_DIR}/patch/hypre-hip-rocsparse-sort.patch" + "|${CMAKE_SOURCE_DIR}/patch/hypre-hip-standard-ilu.patch" ) + endif() + + string( REPLACE ";" "" HYPRE_PATCH_FILES "${HYPRE_PATCH_FILES}" ) + set( HYPRE_PATCH_COMMAND + ${CMAKE_COMMAND} + "-DGEOS_PATCH_STRIP:STRING=1" + "-DGEOS_PATCH_FILES:STRING=${HYPRE_PATCH_FILES}" + -P ${GEOS_TPL_PATCH_SCRIPT} ) message(STATUS "Building HYPRE found at ${HYPRE_URL}") @@ -1088,6 +1134,7 @@ if( ENABLE_HYPRE ) "-DCMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES}" "-DCMAKE_HIP_STANDARD:STRING=${HYPRE_HIP_STANDARD}" "-DCMAKE_HIP_STANDARD_REQUIRED:BOOL=ON" + "-DCMAKE_HIP_FLAGS:STRING=${CMAKE_HIP_FLAGS}" "-DROCM_PATH:PATH=${ROCM_PATH}" ) endif() @@ -1099,7 +1146,7 @@ if( ENABLE_HYPRE ) list( APPEND HYPRE_CMAKE_ARGS "-DHYPRE_ENABLE_GPU_PROFILING:BOOL=ON" ) endif() - if( BUILD_SHARED_LIBS ) + if( CHAI_BUILD_SHARED_LIBS ) set( HYPRE_UMPIRE_LIBRARY_SUFFIX so ) else() set( HYPRE_UMPIRE_LIBRARY_SUFFIX a ) @@ -1146,6 +1193,7 @@ if( ENABLE_HYPRE ) SOURCE_SUBDIR src INSTALL_DIR ${HYPRE_DIR} DEPENDS ${HYPRE_DEPENDS} + PATCH_COMMAND ${HYPRE_PATCH_COMMAND} LIST_SEPARATOR "^" CMAKE_GENERATOR ${TPL_GENERATOR} CMAKE_ARGS ${HYPRE_CMAKE_ARGS} @@ -1168,12 +1216,19 @@ if( ENABLE_HYPREDRV ) set( HYPREDRV_DIR "${CMAKE_INSTALL_PREFIX}/hypredrive" ) set( HYPREDRV_URL "https://github.com/hypre-space/hypredrive/archive/00cdb36800d6754b01467a4ef62e0e4ae19e6897.tar.gz" ) set( HYPREDRV_URL_HASH "c33212d5fb7391c81ae22ae801d387f41c2c3f293f48ccd8153845d97ba5150c" ) + # The static SuperLU dependency is C++, while hypredrive-cli is a C target. + # Patch the target to use the C++ linker so its runtime is selected by CMake. + set( HYPREDRV_PATCH_FILES "${CMAKE_SOURCE_DIR}/patch/hypredrive-cxx-linker.patch" ) set( HYPREDRV_C_COMPILER ${CMAKE_C_COMPILER} ) set( HYPREDRV_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" ) set( HYPREDRV_HYPRE_INCLUDE_DIRS "${HYPRE_DIR}/include" ) set( HYPREDRV_HYPRE_LIBRARY "${HYPRE_DIR}/lib/libHYPRE.a" ) set( HYPREDRV_HYPRE_LIBRARIES "${HYPREDRV_HYPRE_LIBRARY}" ) list( APPEND HYPREDRV_HYPRE_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} ) + set( HYPREDRV_HYPRE_USE_OPENMP OFF ) + if( ENABLE_OPENMP ) + set( HYPREDRV_HYPRE_USE_OPENMP ON ) + endif() if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) list( APPEND HYPREDRV_HYPRE_LIBRARIES "${SUPERLU_DIR}/lib/libsuperlu_dist.a" @@ -1195,13 +1250,18 @@ if( ENABLE_HYPREDRV ) if( ENABLE_HYPRE_DEVICE STREQUAL "CUDA" ) set( HYPREDRV_ENABLE_CUDA ON ) # The legacy HYPRE build uses Umpire for device memory. Its generated - # headers include Umpire headers, and its static archive leaves the - # Umpire/CUDA dependencies to consumers, so propagate both explicitly. + # headers include Umpire headers, and its archive leaves the Umpire/CUDA + # dependencies to consumers, so propagate both explicitly. + if( CHAI_BUILD_SHARED_LIBS ) + set( HYPREDRV_UMPIRE_LIBRARY_SUFFIX so ) + else() + set( HYPREDRV_UMPIRE_LIBRARY_SUFFIX a ) + endif() list( APPEND HYPREDRV_HYPRE_INCLUDE_DIRS "${CHAI_DIR}/include" ) list( APPEND HYPREDRV_HYPRE_LIBRARIES - "${CHAI_DIR}/lib/libumpire.a" - "${CHAI_DIR}/lib/libcamp.a" - "${CHAI_DIR}/lib/libfmt.a" + "${CHAI_DIR}/lib/libumpire.${HYPREDRV_UMPIRE_LIBRARY_SUFFIX}" + "${CHAI_DIR}/lib/libcamp.${HYPREDRV_UMPIRE_LIBRARY_SUFFIX}" + "${CHAI_DIR}/lib/libfmt.${HYPREDRV_UMPIRE_LIBRARY_SUFFIX}" "-L${CUDA_TOOLKIT_ROOT_DIR}/lib64" -lcudart -lcusparse -lcublas -lcurand -lcusolver -ldl -lstdc++ ) @@ -1211,11 +1271,18 @@ if( ENABLE_HYPREDRV ) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} ) elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) set( HYPREDRV_ENABLE_HIP ON ) + string( APPEND HYPREDRV_PATCH_FILES + "|${CMAKE_SOURCE_DIR}/patch/hypredrive-hip-rocsparse.patch" ) + if( CHAI_BUILD_SHARED_LIBS ) + set( HYPREDRV_UMPIRE_LIBRARY_SUFFIX so ) + else() + set( HYPREDRV_UMPIRE_LIBRARY_SUFFIX a ) + endif() list( APPEND HYPREDRV_HYPRE_INCLUDE_DIRS "${CHAI_DIR}/include" ) list( APPEND HYPREDRV_HYPRE_LIBRARIES - "${CHAI_DIR}/lib/libumpire.a" - "${CHAI_DIR}/lib/libcamp.a" - "${CHAI_DIR}/lib/libfmt.a" + "${CHAI_DIR}/lib/libumpire.${HYPREDRV_UMPIRE_LIBRARY_SUFFIX}" + "${CHAI_DIR}/lib/libcamp.${HYPREDRV_UMPIRE_LIBRARY_SUFFIX}" + "${CHAI_DIR}/lib/libfmt.${HYPREDRV_UMPIRE_LIBRARY_SUFFIX}" "-L${ROCM_PATH}/lib" -lrocblas -lrocsparse -lrocrand -lrocsolver -lamdhip64 -ldl -lstdc++ ) @@ -1225,6 +1292,12 @@ if( ENABLE_HYPREDRV ) -D ROCM_PATH:PATH=${ROCM_PATH} ) endif() + set( HYPREDRV_PATCH_COMMAND + ${CMAKE_COMMAND} + "-DGEOS_PATCH_STRIP:STRING=1" + "-DGEOS_PATCH_FILES:STRING=${HYPREDRV_PATCH_FILES}" + -P ${GEOS_TPL_PATCH_SCRIPT} ) + set( HYPREDRV_CALIPER_ARGS ) set( HYPREDRV_ENABLE_CALIPER OFF ) if( DEFINED ENABLE_CALIPER_HYPRE AND ENABLE_CALIPER_HYPRE ) @@ -1251,6 +1324,7 @@ if( ENABLE_HYPREDRV ) PREFIX ${PROJECT_BINARY_DIR}/hypredrive INSTALL_DIR ${HYPREDRV_DIR} DEPENDS hypre + PATCH_COMMAND ${HYPREDRV_PATCH_COMMAND} BUILD_COMMAND ${TPL_BUILD_COMMAND} INSTALL_COMMAND ${TPL_INSTALL_COMMAND} CMAKE_GENERATOR ${TPL_GENERATOR} @@ -1273,6 +1347,7 @@ if( ENABLE_HYPREDRV ) -D HYPREDRV_ENABLE_CALIPER:BOOL=${HYPREDRV_ENABLE_CALIPER} -D HYPREDRV_ENABLE_CUDA:BOOL=${HYPREDRV_ENABLE_CUDA} -D HYPREDRV_ENABLE_HIP:BOOL=${HYPREDRV_ENABLE_HIP} + -D HYPREDRV_HYPRE_USE_OPENMP:BOOL=${HYPREDRV_HYPRE_USE_OPENMP} ${HYPREDRV_MPI_ARGS} ${HYPREDRV_ACCELERATOR_ARGS} ${HYPREDRV_CALIPER_ARGS} ) @@ -1298,9 +1373,9 @@ if(ENABLE_SUITESPARSE) set( SUITESPARSE_LONG_INT_FLAGS "-D\'SuiteSparse_long=long long int\' -DSuiteSparse_long_max=2147483647 -DSuiteSparse_long_idd=\\\"d\\\" -D\'SuiteSparse_long_id=\\\"%d\\\"\'" ) endif() - set( SUITESPARSE_C_FLAGS "${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" ) - set( SUITESPARSE_CXX_FLAGS "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}" ) - set( SUITESPARSE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_${BUILD_TYPE_UPPER}}" ) + set( SUITESPARSE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}}" ) + set( SUITESPARSE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}" ) + set( SUITESPARSE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS_${BUILD_TYPE_UPPER}}" ) set( SUITESPARSE_C_COMPILER ${CMAKE_C_COMPILER} ) set( SUITESPARSE_CXX_COMPILER ${CMAKE_CXX_COMPILER} ) @@ -1310,12 +1385,24 @@ if(ENABLE_SUITESPARSE) set(SUITESPARSE_OPENMP_FLAG "${OpenMP_C_FLAGS}") endif() + set(SUITESPARSE_LINK_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS}") + # SuiteSparse_config.mk appends its install-library search path with + # `LDFLAGS +=`. Since LDFLAGS is supplied on the make command line, GNU + # Make does not apply that append. Add the staging library directory here + # so the subsequently built SuiteSparse components can find + # libsuitesparseconfig before the final install step. + string(APPEND SUITESPARSE_LINK_FLAGS " -L${PROJECT_BINARY_DIR}/suitesparse/src/suitesparse/lib") + if (SUITESPARSE_OPENMP_FLAG) + string(APPEND SUITESPARSE_LINK_FLAGS " ${SUITESPARSE_OPENMP_FLAG}") + endif() + set( SUITESPARSE_CONFIG_ARGS "CC=${SUITESPARSE_C_COMPILER} \ CFLAGS=\"${SUITESPARSE_C_FLAGS} ${SUITESPARSE_LONG_INT_FLAGS}\" \ CXX=${SUITESPARSE_CXX_COMPILER} \ CXXFLAGS=\"${SUITESPARSE_CXX_FLAGS}\" \ FC=${SUITESPARSE_Fortran_COMPILER} \ FCFLAGS=\"${SUITESPARSE_Fortran_FLAGS}\" \ +LDFLAGS=\"${SUITESPARSE_LINK_FLAGS}\" \ JOBS=${NUM_PROC} \ CFOPENMP=${SUITESPARSE_OPENMP_FLAG} \ INSTALL=${SUITESPARSE_DIR} \ @@ -1399,6 +1486,7 @@ if( ENABLE_SCOTCH ) -D SCOTCH_DETERMINISTIC:STRING=FULL -D CMAKE_C_COMPILER=${MPI_C_COMPILER} -D CMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} + -D CMAKE_Fortran_COMPILER:FILEPATH=${CMAKE_Fortran_COMPILER} -D CMAKE_C_FLAGS=${C_FLAGS_NO_WARNINGS} -D CMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} -D CMAKE_CXX_FLAGS=${CXX_FLAGS_NO_WARNINGS} @@ -1539,8 +1627,10 @@ endif( ENABLE_MPI ) URL ${VTK_URL} URL_HASH SHA256=${VTK_URL_HASH} INSTALL_DIR ${VTK_DIR} - PATCH_COMMAND patch --forward -p0 < ${CMAKE_SOURCE_DIR}/patch/vtkCellGridReader.cxx.patch && - patch --forward -p0 < ${CMAKE_SOURCE_DIR}/patch/vtkLegacyCellGridReader.cxx.patch + PATCH_COMMAND ${CMAKE_COMMAND} + "-DGEOS_PATCH_STRIP:STRING=0" + "-DGEOS_PATCH_FILES:STRING=${CMAKE_SOURCE_DIR}/patch/vtkCellGridReader.cxx.patch|${CMAKE_SOURCE_DIR}/patch/vtkLegacyCellGridReader.cxx.patch" + -P ${GEOS_TPL_PATCH_SCRIPT} BUILD_COMMAND ${TPL_BUILD_COMMAND} INSTALL_COMMAND "${TPL_INSTALL_COMMAND}" CMAKE_GENERATOR ${TPL_GENERATOR} @@ -1616,6 +1706,7 @@ message(STATUS "Building = ${build_list}") # the `tpl` target after all external projects have been built. set(GEOS_TPL_HOST_CONFIG_TPL_DIR "${CMAKE_INSTALL_PREFIX}") set(GEOS_TPL_HOST_CONFIG_CAMP_DIR "${CMAKE_INSTALL_PREFIX}/raja") +set(GEOS_TPL_HOST_CONFIG_FMT_DIR "${FMT_DIR}/lib/cmake/fmt") # Include the toolchain identity in the host-config name so that an install # directory makes its compiler/backend requirements visible without opening diff --git a/cmake/apply-patches.cmake b/cmake/apply-patches.cmake new file mode 100644 index 00000000..37cd38f1 --- /dev/null +++ b/cmake/apply-patches.cmake @@ -0,0 +1,52 @@ +# Apply one or more unified patches without failing when a patch is already +# present in an incremental ExternalProject build. +if( NOT DEFINED GEOS_PATCH_FILES OR GEOS_PATCH_FILES STREQUAL "" ) + message( FATAL_ERROR "GEOS_PATCH_FILES must name at least one patch file." ) +endif() + +if( NOT DEFINED GEOS_PATCH_STRIP OR GEOS_PATCH_STRIP STREQUAL "" ) + set( GEOS_PATCH_STRIP 1 ) +endif() + +# A pipe is used as the list separator because semicolons are list separators +# in CMake command-line definitions. +string( REPLACE "|" ";" _geos_patch_files "${GEOS_PATCH_FILES}" ) + +foreach( _geos_patch_file IN LISTS _geos_patch_files ) + if( NOT EXISTS "${_geos_patch_file}" ) + message( FATAL_ERROR "Patch file does not exist: ${_geos_patch_file}" ) + endif() + + execute_process( + COMMAND patch --batch --forward --dry-run -p${GEOS_PATCH_STRIP} -i "${_geos_patch_file}" + RESULT_VARIABLE _geos_patch_forward_status + OUTPUT_VARIABLE _geos_patch_forward_output + ERROR_VARIABLE _geos_patch_forward_error ) + + if( _geos_patch_forward_status EQUAL 0 ) + execute_process( + COMMAND patch --batch --forward -p${GEOS_PATCH_STRIP} -i "${_geos_patch_file}" + RESULT_VARIABLE _geos_patch_apply_status + OUTPUT_VARIABLE _geos_patch_apply_output + ERROR_VARIABLE _geos_patch_apply_error ) + if( NOT _geos_patch_apply_status EQUAL 0 ) + message( FATAL_ERROR + "Failed to apply patch ${_geos_patch_file}:\n" + "${_geos_patch_apply_output}${_geos_patch_apply_error}" ) + endif() + else() + execute_process( + COMMAND patch --batch --reverse --dry-run -p${GEOS_PATCH_STRIP} -i "${_geos_patch_file}" + RESULT_VARIABLE _geos_patch_reverse_status + OUTPUT_VARIABLE _geos_patch_reverse_output + ERROR_VARIABLE _geos_patch_reverse_error ) + if( _geos_patch_reverse_status EQUAL 0 ) + message( STATUS "Patch already applied: ${_geos_patch_file}" ) + else() + message( FATAL_ERROR + "Patch cannot be applied or recognized as already applied: ${_geos_patch_file}\n" + "Forward check:\n${_geos_patch_forward_output}${_geos_patch_forward_error}\n" + "Reverse check:\n${_geos_patch_reverse_output}${_geos_patch_reverse_error}" ) + endif() + endif() +endforeach() diff --git a/cmake/geos-host-config.cmake.in b/cmake/geos-host-config.cmake.in index 26e36ed3..6b464f7b 100644 --- a/cmake/geos-host-config.cmake.in +++ b/cmake/geos-host-config.cmake.in @@ -39,7 +39,7 @@ set(ADIAK_DIR "@ADIAK_DIR@" CACHE PATH "") set(CALIPER_DIR "@CALIPER_DIR@" CACHE PATH "") set(PUGIXML_DIR "@PUGIXML_DIR@" CACHE PATH "") set(VTK_DIR "@VTK_DIR@" CACHE PATH "") -set(FMT_DIR "@FMT_DIR@" CACHE PATH "") +set(FMT_DIR "@GEOS_TPL_HOST_CONFIG_FMT_DIR@" CACHE PATH "") set(METIS_DIR "@METIS_DIR@" CACHE PATH "") set(PARMETIS_DIR "@PARMETIS_DIR@" CACHE PATH "") set(SUPERLU_DIST_DIR "@SUPERLU_DIR@" CACHE PATH "") @@ -110,6 +110,7 @@ if("@ENABLE_HIP@" STREQUAL "ON") set(CMAKE_HIP_STANDARD_REQUIRED ON CACHE BOOL "") set(ROCM_PATH "@ROCM_PATH@" CACHE PATH "") set(HIP_ROOT "@HIP_ROOT@" CACHE PATH "") + set(CMAKE_HIP_FLAGS "@CMAKE_HIP_FLAGS@" CACHE STRING "") set(GPU_TARGETS "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "") set(AMDGPU_TARGETS "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "") endif() diff --git a/patch/hypre-hip-rocsparse-sort.patch b/patch/hypre-hip-rocsparse-sort.patch new file mode 100644 index 00000000..14e25e5f --- /dev/null +++ b/patch/hypre-hip-rocsparse-sort.patch @@ -0,0 +1,24 @@ +--- a/src/seq_mv/csr_matop_device.c ++++ b/src/seq_mv/csr_matop_device.c +@@ -4016,6 +4016,21 @@ + void *pBuffer = NULL; + HYPRE_Int *P = NULL; + HYPRE_Complex *d_a_tmp; ++#if defined(HYPRE_USING_HIP) ++ /* ++ * ROCm 7.2's legacy rocSPARSE csrsort path poisons the subsequent ++ * csritilu0 preprocessing on gfx1100. Use HYPRE's device-side stable ++ * tuple sort instead. The row index is a secondary key; row pointers ++ * remain valid because entries from different rows are never exchanged. ++ */ ++ HYPRE_Int *d_row_ind = hypre_CsrRowPtrsToIndicesDevice(n, num_nonzeros, ++ (HYPRE_Int *) d_ia); ++ hypre_StableSortByTupleKeyDevice(num_nonzeros, d_row_ind, d_ja_sorted, ++ d_a_sorted, 0); ++ hypre_ForceSyncComputeStream(); ++ hypre_TFree(d_row_ind, HYPRE_MEMORY_DEVICE); ++ return hypre_error_flag; ++#endif + + // FIXME: There is not in-place version of csr sort in rocSPARSE currently, so we make + // a temporary copy of the data for gthr, sort that, and then copy the sorted values diff --git a/patch/hypre-hip-standard-ilu.patch b/patch/hypre-hip-standard-ilu.patch new file mode 100644 index 00000000..86c662e0 --- /dev/null +++ b/patch/hypre-hip-standard-ilu.patch @@ -0,0 +1,28 @@ +--- a/src/parcsr_ls/par_ilu_setup_device.c ++++ b/src/parcsr_ls/par_ilu_setup_device.c +@@ -161,1 +161,1 @@ +-#if !defined(HYPRE_USING_SYCL) ++#if !defined(HYPRE_USING_SYCL) && !defined(HYPRE_USING_HIP) +@@ -187,0 +188,15 @@ ++#if defined(HYPRE_USING_HIP) ++ /* ++ * rocSPARSE's legacy csrilu0 analysis is not functional on the ++ * gfx1100 devices shipped with ROCm 7.2. Use HYPRE's classical ++ * ILU(0) implementation instead. The factorization is performed ++ * on the host only during setup and the resulting LDU factors are ++ * migrated back to the device by hypre_ILUSetupLDUtoVendor. ++ */ ++ if (fill_level == 0 && !(ilu_type % 10)) ++ { ++ hypre_ParCSRMatrixMigrate(Apq, HYPRE_MEMORY_HOST); ++ hypre_ILUSetupILU0(Apq, NULL, NULL, n, n, ++ &parL, &parD, &parU, &parS, &uend); ++ } ++#endif +@@ -214,0 +215,6 @@ ++#if defined(HYPRE_USING_HIP) ++ else if (fill_level == 0 && !(ilu_type % 10)) ++ { ++ hypre_TFree(parD, HYPRE_MEMORY_HOST); ++ } ++#endif diff --git a/patch/hypre-mixedint-export.patch b/patch/hypre-mixedint-export.patch new file mode 100644 index 00000000..a6b5554b --- /dev/null +++ b/patch/hypre-mixedint-export.patch @@ -0,0 +1,12 @@ +diff --git a/src/parcsr_mv/par_csr_matrix.c b/src/parcsr_mv/par_csr_matrix.c +--- a/src/parcsr_mv/par_csr_matrix.c ++++ b/src/parcsr_mv/par_csr_matrix.c +@@ -2458,1 +2458,1 @@ +- HYPRE_Int contact_send_buf[1]; ++ HYPRE_BigInt contact_send_buf[1]; +@@ -2504,1 +2504,1 @@ +- contact_send_buf[0] = (HYPRE_Int) hypre_ParCSRMatrixLastRowIndex(par_matrix); ++ contact_send_buf[0] = hypre_ParCSRMatrixLastRowIndex(par_matrix); +@@ -2533,1 +2533,1 @@ +- contact_send_buf_starts, sizeof(HYPRE_Int), ++ contact_send_buf_starts, sizeof(HYPRE_BigInt), diff --git a/patch/hypre-rcm-capacity.patch b/patch/hypre-rcm-capacity.patch new file mode 100644 index 00000000..f28022ba --- /dev/null +++ b/patch/hypre-rcm-capacity.patch @@ -0,0 +1,7 @@ +diff --git a/src/parcsr_ls/par_ilu.c b/src/parcsr_ls/par_ilu.c +--- a/src/parcsr_ls/par_ilu.c ++++ b/src/parcsr_ls/par_ilu.c +@@ -2579,1 +2579,2 @@ +- G_capacity = hypre_max((A_nnz * n * n / num_nodes / num_nodes) - num_nodes, 1); ++ G_capacity = hypre_max((HYPRE_Int)((HYPRE_Real) A_nnz * n * n / ++ num_nodes / num_nodes) - num_nodes, 1); diff --git a/patch/hypre-tagged-innerprod.patch b/patch/hypre-tagged-innerprod.patch new file mode 100644 index 00000000..61c62bb0 --- /dev/null +++ b/patch/hypre-tagged-innerprod.patch @@ -0,0 +1,8 @@ +diff --git a/src/seq_mv/vector.c b/src/seq_mv/vector.c +--- a/src/seq_mv/vector.c ++++ b/src/seq_mv/vector.c +@@ -1384,1 +1384,1 @@ +- num_threads * num_tags, ++ num_threads * (num_tags + 1), +@@ -1413,0 +1414,1 @@ ++ hypre_TFree(thread_sums, HYPRE_MEMORY_HOST); diff --git a/patch/hypre-umpire-wrapper-lifetime.patch b/patch/hypre-umpire-wrapper-lifetime.patch new file mode 100644 index 00000000..93eae9da --- /dev/null +++ b/patch/hypre-umpire-wrapper-lifetime.patch @@ -0,0 +1,87 @@ +diff --git a/src/utilities/umpire.c b/src/utilities/umpire.c +--- a/src/utilities/umpire.c ++++ b/src/utilities/umpire.c +@@ -44,0 +45,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -140,0 +141,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -202,0 +203,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -265,0 +266,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -354,43 +358,63 @@ hypre_UmpireFinalize(hypre_Handle *hypre_handle_) + umpire_allocator allocator; + + #if defined(HYPRE_USING_UMPIRE_HOST) +- if (hypre_HandleUmpireOwnHostPool(hypre_handle_)) ++ if (hypre_HandleUmpireHostAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpireHostPool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnHostPool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpireHostPool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpireHostPool(hypre_handle_)); + } + hypre_HandleUmpireHostAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpireHostAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnHostPool(hypre_handle_) = 0; + #endif + + #if defined(HYPRE_USING_UMPIRE_DEVICE) +- if (hypre_HandleUmpireOwnDevicePool(hypre_handle_)) ++ if (hypre_HandleUmpireDeviceAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpireDevicePool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnDevicePool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpireDevicePool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpireDevicePool(hypre_handle_)); + } + hypre_HandleUmpireDeviceAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpireDeviceAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnDevicePool(hypre_handle_) = 0; + #endif + + #if defined(HYPRE_USING_UMPIRE_UM) +- if (hypre_HandleUmpireOwnUMPool(hypre_handle_)) ++ if (hypre_HandleUmpireUMAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpireUMPool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnUMPool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpireUMPool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpireUMPool(hypre_handle_)); + } + hypre_HandleUmpireUMAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpireUMAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnUMPool(hypre_handle_) = 0; + #endif + + #if defined(HYPRE_USING_UMPIRE_PINNED) +- if (hypre_HandleUmpireOwnPinnedPool(hypre_handle_)) ++ if (hypre_HandleUmpirePinnedAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpirePinnedPool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnPinnedPool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpirePinnedPool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpirePinnedPool(hypre_handle_)); + } + hypre_HandleUmpirePinnedAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpirePinnedAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnPinnedPool(hypre_handle_) = 0; + #endif + + return hypre_error_flag; diff --git a/patch/hypredrive-cxx-linker.patch b/patch/hypredrive-cxx-linker.patch new file mode 100644 index 00000000..8d4adf66 --- /dev/null +++ b/patch/hypredrive-cxx-linker.patch @@ -0,0 +1,25 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -235,6 +235,12 @@ + $ + INTERFACE + $) ++if(HYPREDRV_HYPRE_USE_OPENMP) ++ find_package(OpenMP REQUIRED COMPONENTS C) ++ if(TARGET OpenMP::OpenMP_C) ++ target_link_libraries(HYPREDRV PUBLIC OpenMP::OpenMP_C) ++ endif() ++endif() + + if(HYPREDRV_INSTRUMENTATION_LINK_FLAGS) + target_link_options(HYPREDRV PUBLIC ${HYPREDRV_INSTRUMENTATION_LINK_FLAGS}) +@@ -419,4 +425,9 @@ + # Define the main executable (hypredrive-cli) + add_executable(hypredrive-cli src/internal/main.c) ++# HYPRE's static dependency graph can contain C++ libraries such as ++# SuperLU_DIST. Select the C++ linker so the C++ runtime is linked as well. ++if(CMAKE_CXX_COMPILER) ++ set_property(TARGET hypredrive-cli PROPERTY LINKER_LANGUAGE CXX) ++endif() + target_link_libraries(hypredrive-cli PRIVATE ${HYPREDRV_INTERNAL_LINK_TARGET}) + hypredrv_add_hypre_build_dependency(hypredrive-cli) diff --git a/patch/hypredrive-hip-rocsparse.patch b/patch/hypredrive-hip-rocsparse.patch new file mode 100644 index 00000000..827afc5b --- /dev/null +++ b/patch/hypredrive-hip-rocsparse.patch @@ -0,0 +1,77 @@ +--- a/src/internal/amg.c ++++ b/src/internal/amg.c +@@ -190,19 +190,29 @@ + *-----------------------------------------------------------------------------*/ + + void + hypredrv_AMGrlxSetDefaultArgs(AMGrlx_args *args) + { ++#ifdef HYPRE_USING_HIP ++ /* Use a device-compatible smoother even when AMG does not coarsen. */ ++ args->type = 18; ++#else + args->type = -1; ++#endif + #ifdef HYPRE_USING_GPU + args->down_type = 18; + args->up_type = 18; + #else + args->down_type = 13; + args->up_type = 14; + #endif ++#ifdef HYPRE_USING_HIP ++ /* Avoid the legacy ROCm CSR triangular-solve path on HIP GPUs. */ ++ args->coarse_type = 18; ++#else + args->coarse_type = 9; ++#endif + args->down_sweeps = -1; + args->up_sweeps = -1; + args->coarse_sweeps = 1; + args->num_sweeps = 1; + args->order = 0; +@@ -1187,7 +1193,12 @@ + #else + HYPRE_BoomerAMGSetILULocalReordering(precon, args->smoother.ilu.reordering); + #endif ++#ifdef HYPRE_USING_HIP ++ /* Keep the classical HIP ILU factors off rocSPARSE csrsv. */ ++ HYPRE_BoomerAMGSetILUTriSolve(precon, 0); ++#else + HYPRE_BoomerAMGSetILUTriSolve(precon, args->smoother.ilu.tri_solve); ++#endif + HYPRE_BoomerAMGSetILULowerJacobiIters(precon, args->smoother.ilu.lower_jac_iters); + HYPRE_BoomerAMGSetILUUpperJacobiIters(precon, args->smoother.ilu.upper_jac_iters); + #endif +--- a/src/internal/ilu.c ++++ b/src/internal/ilu.c +@@ -87,7 +87,12 @@ + HYPRE_ILUSetLocalReordering(precon, args->reordering); + #endif + #if HYPRE_CHECK_MIN_VERSION(22600, 0) ++#ifdef HYPRE_USING_HIP ++ /* HYPRE uses classical host ILU(0) setup on HIP; avoid rocSPARSE csrsv. */ ++ HYPRE_ILUSetTriSolve(precon, 0); ++#else + HYPRE_ILUSetTriSolve(precon, args->tri_solve); ++#endif + HYPRE_ILUSetLowerJacobiIters(precon, args->lower_jac_iters); + HYPRE_ILUSetUpperJacobiIters(precon, args->upper_jac_iters); + #endif +--- a/src/HYPREDRV.c ++++ b/src/HYPREDRV.c +@@ -313,10 +313,13 @@ + #endif + + #if HYPRE_CHECK_MIN_VERSION(22500, 0) + HYPRE_SetSpGemmUseVendor(hypredrv->iargs->general.use_vendor_spgemm); + HYPRE_SetSpMVUseVendor(hypredrv->iargs->general.use_vendor_spmv); ++#ifdef HYPRE_USING_HIP ++ HYPRE_SetSpTransUseVendor(0); ++#endif + #endif + + #ifdef HYPRE_USING_UMPIRE + HYPRE_SetUmpireDevicePoolName("HYPRE_DEVICE"); + HYPRE_SetUmpireUMPoolName("HYPRE_UM"); + HYPRE_SetUmpireHostPoolName("HYPRE_HOST"); diff --git a/patch/raja-reducer-assignment.patch b/patch/raja-reducer-assignment.patch new file mode 100644 index 00000000..85ad8dc8 --- /dev/null +++ b/patch/raja-reducer-assignment.patch @@ -0,0 +1,94 @@ +diff --git a/include/RAJA/policy/cuda/reduce.hpp b/include/RAJA/policy/cuda/reduce.hpp +--- a/include/RAJA/policy/cuda/reduce.hpp ++++ b/include/RAJA/policy/cuda/reduce.hpp +@@ -1123,6 +1123,43 @@ + #endif + } + ++ //! assign a reducer while preserving ownership of host and device resources ++ Reduce& operator=(const Reduce& other) ++ { ++ if (this != &other) ++ { ++ // The implicitly generated assignment operator shallow-copies the ++ // reducer state. In particular, it overwrites an owning tally pointer ++ // and leaks the tally allocated by the destination. It also copies the ++ // device-ownership flag, which can make two reducers free the same ++ // device allocation. Match the copy constructor's non-owning semantics ++ // after releasing resources currently owned by the destination. ++ if (parent == this) ++ { ++ delete tally_or_val_ptr.list; ++ } ++ else if (parent == nullptr) ++ { ++ val.teardownForDevice(); ++ } ++ ++ parent = other.parent; ++ tally_or_val_ptr = other.tally_or_val_ptr; ++ val = reduce_data_type(other.val); ++ ++ if (parent && tally_or_val_ptr.list) ++ { ++ if (val.setupForDevice()) ++ { ++ tally_or_val_ptr.val_ptr = val.init_grid_vals( ++ tally_or_val_ptr.list->new_value(currentResource())); ++ parent = nullptr; ++ } ++ } ++ } ++ return *this; ++ } ++ + //! apply reduction upon destruction and cleanup resources owned by this copy + // on device store in pinned buffer on host + RAJA_HOST_DEVICE +diff --git a/include/RAJA/policy/hip/reduce.hpp b/include/RAJA/policy/hip/reduce.hpp +--- a/include/RAJA/policy/hip/reduce.hpp ++++ b/include/RAJA/policy/hip/reduce.hpp +@@ -1116,6 +1116,43 @@ + #endif + } + ++ //! assign a reducer while preserving ownership of host and device resources ++ Reduce& operator=(const Reduce& other) ++ { ++ if (this != &other) ++ { ++ // The implicitly generated assignment operator shallow-copies the ++ // reducer state. In particular, it overwrites an owning tally pointer ++ // and leaks the tally allocated by the destination. It also copies the ++ // device-ownership flag, which can make two reducers free the same ++ // device allocation. Match the copy constructor's non-owning semantics ++ // after releasing resources currently owned by the destination. ++ if (parent == this) ++ { ++ delete tally_or_val_ptr.list; ++ } ++ else if (parent == nullptr) ++ { ++ val.teardownForDevice(); ++ } ++ ++ parent = other.parent; ++ tally_or_val_ptr = other.tally_or_val_ptr; ++ val = reduce_data_type(other.val); ++ ++ if (parent && tally_or_val_ptr.list) ++ { ++ if (val.setupForDevice()) ++ { ++ tally_or_val_ptr.val_ptr = val.init_grid_vals( ++ tally_or_val_ptr.list->new_value(currentResource())); ++ parent = nullptr; ++ } ++ } ++ } ++ return *this; ++ } ++ + //! apply reduction upon destruction and cleanup resources owned by this copy + // on device store in pinned buffer on host + RAJA_HOST_DEVICE diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 7f57cbfc..9b851d4e 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -83,7 +83,6 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('cuda_stack_size', default="0", description="Defines the adjusted cuda stack \ size limit if required. Zero or negative keep default behavior") - # SPHINX_BEGIN_DEPENDS depends_on("c", type="build") depends_on("cxx", type="build") @@ -134,7 +133,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on('umpire+cuda cuda_arch={0}'.format(sm_), when='cuda_arch={0}'.format(sm_)) depends_on('chai+cuda~separable_compilation cuda_arch={0}'.format(sm_), when='cuda_arch={0}'.format(sm_)) depends_on('camp+cuda cuda_arch={0}'.format(sm_), when='cuda_arch={0}'.format(sm_)) - depends_on('hypre+cuda cuda_arch={0}'.format(sm_), when='cuda_arch={0}'.format(sm_)) + depends_on('hypre@develop+cuda cuda_arch={0}'.format(sm_), when='cuda_arch={0}'.format(sm_)) with when('+rocm'): for gfx_ in ROCmPackage.amdgpu_targets: @@ -142,7 +141,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on(f"umpire+rocm amdgpu_target={gfx_}", when=f"amdgpu_target={gfx_}") depends_on(f"chai+rocm~separable_compilation amdgpu_target={gfx_}", when=f"amdgpu_target={gfx_}") depends_on(f"camp+rocm amdgpu_target={gfx_}", when=f"amdgpu_target={gfx_}") - depends_on(f"hypre+rocm amdgpu_target={gfx_}", when=f"amdgpu_target={gfx_}") + depends_on(f"hypre@develop+rocm amdgpu_target={gfx_}", when=f"amdgpu_target={gfx_}") # # IO @@ -174,8 +173,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("superlu-dist+openmp", when="+openmp") # -Wno-error=implicit-function-declaration needed for 'METIS_PartMeshDual' error - depends_on("scotch@7.0.8 ~compression +mpi +esmumps +int64 determinism=FULL ~shared ~metis build_system=cmake cflags='-fPIC' cxxflags='-fPIC'", when='+scotch') - + depends_on("scotch@7.0.8 ~compression +mpi +esmumps +int64 determinism=FULL ~metis build_system=cmake cflags='-fPIC' cxxflags='-fPIC'", when='+scotch') depends_on('suite-sparse@5.10.1') depends_on("suite-sparse~openmp", when="~openmp") depends_on("suite-sparse+openmp", when="+openmp") @@ -188,19 +186,19 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("trilinos+openmp", when="+openmp") with when("+hypre"): - depends_on("hypre +superlu-dist+mixedint+mpi", when='~cuda~rocm') - depends_on("hypre +cuda+superlu-dist+mixedint+mpi+umpire~unified-memory", when='+cuda') - depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire~unified-memory", when='+rocm') - depends_on("hypre ~openmp", when="~openmp") - depends_on("hypre +caliper", when="+caliper") - depends_on("hypre +pic", when="~shared") - depends_on("hypre +shared", when="+shared") + depends_on("hypre@develop +superlu-dist+mixedint+mpi", when='~cuda~rocm') + depends_on("hypre@develop +cuda+superlu-dist+mixedint+mpi+umpire~unified-memory", when='+cuda') + depends_on("hypre@develop +rocm+superlu-dist+mixedint+mpi+umpire~unified-memory", when='+rocm') + depends_on("hypre@develop ~openmp", when="~openmp") + depends_on("hypre@develop +caliper", when="+caliper") + depends_on("hypre@develop +pic", when="~shared") + depends_on("hypre@develop +shared", when="+shared") with when("+hypredrive"): - depends_on("hypredrive +superlu-dist") - depends_on("hypredrive +pic", when="~shared") - depends_on("hypredrive +shared", when="+shared") - depends_on("hypredrive +caliper", when="+caliper") + depends_on("hypredrive@develop +superlu-dist") + depends_on("hypredrive@develop +pic", when="~shared") + depends_on("hypredrive@develop +shared", when="+shared") + depends_on("hypredrive@develop +caliper", when="+caliper") depends_on('petsc@3.19.4~hdf5~hypre+int64', when='+petsc') depends_on('petsc+ptscotch', when='+petsc+scotch') @@ -285,7 +283,6 @@ def _get_host_config_path(self, spec, lvarray=False): str(spec.compiler.name), str(spec.compiler.version), gpu_backend) - dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) return host_config_path @@ -451,7 +448,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) - cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' + cmake_cuda_flags = ('-restrict --allow-unsupported-compiler --extended-lambda -Werror ' 'cross-execution-space-call,reorder,' 'deprecated-declarations') @@ -772,7 +769,7 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) - cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' + cmake_cuda_flags = ('-restrict --allow-unsupported-compiler --extended-lambda -Werror ' 'cross-execution-space-call,reorder,' 'deprecated-declarations') diff --git a/scripts/spack_packages/packages/hypre/package.py b/scripts/spack_packages/packages/hypre/package.py index 1b29b44b..cb6a41c2 100644 --- a/scripts/spack_packages/packages/hypre/package.py +++ b/scripts/spack_packages/packages/hypre/package.py @@ -9,4 +9,15 @@ class Hypre(BuiltinHypre): # Keep the complete Git history so HYPRE can report its tagged, # distance-aware development version (for example, v3.0.2-2-g). - version("develop", branch="master", get_full_repo=True) + version("develop", branch="master", get_full_repo=True, preferred=True) + + # Keep the source fixes used by the legacy GEOS TPL superbuild in the + # Spack installation path as well. The HIP-only patches are deliberately + # restricted to ROCm builds because they change HYPRE's device setup path. + patch("patches/hypre-mgr-col-lumped-destroy.patch", when="@3.1.0") + patch("patches/hypre-mixedint-export.patch", when="@develop") + patch("patches/hypre-tagged-innerprod.patch", when="@develop") + patch("patches/hypre-rcm-capacity.patch", when="@develop") + patch("patches/hypre-umpire-wrapper-lifetime.patch", when="@develop") + patch("patches/hypre-hip-rocsparse-sort.patch", when="@develop+rocm") + patch("patches/hypre-hip-standard-ilu.patch", when="@develop+rocm") diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-hip-rocsparse-sort.patch b/scripts/spack_packages/packages/hypre/patches/hypre-hip-rocsparse-sort.patch new file mode 100644 index 00000000..14e25e5f --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-hip-rocsparse-sort.patch @@ -0,0 +1,24 @@ +--- a/src/seq_mv/csr_matop_device.c ++++ b/src/seq_mv/csr_matop_device.c +@@ -4016,6 +4016,21 @@ + void *pBuffer = NULL; + HYPRE_Int *P = NULL; + HYPRE_Complex *d_a_tmp; ++#if defined(HYPRE_USING_HIP) ++ /* ++ * ROCm 7.2's legacy rocSPARSE csrsort path poisons the subsequent ++ * csritilu0 preprocessing on gfx1100. Use HYPRE's device-side stable ++ * tuple sort instead. The row index is a secondary key; row pointers ++ * remain valid because entries from different rows are never exchanged. ++ */ ++ HYPRE_Int *d_row_ind = hypre_CsrRowPtrsToIndicesDevice(n, num_nonzeros, ++ (HYPRE_Int *) d_ia); ++ hypre_StableSortByTupleKeyDevice(num_nonzeros, d_row_ind, d_ja_sorted, ++ d_a_sorted, 0); ++ hypre_ForceSyncComputeStream(); ++ hypre_TFree(d_row_ind, HYPRE_MEMORY_DEVICE); ++ return hypre_error_flag; ++#endif + + // FIXME: There is not in-place version of csr sort in rocSPARSE currently, so we make + // a temporary copy of the data for gthr, sort that, and then copy the sorted values diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-hip-standard-ilu.patch b/scripts/spack_packages/packages/hypre/patches/hypre-hip-standard-ilu.patch new file mode 100644 index 00000000..86c662e0 --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-hip-standard-ilu.patch @@ -0,0 +1,28 @@ +--- a/src/parcsr_ls/par_ilu_setup_device.c ++++ b/src/parcsr_ls/par_ilu_setup_device.c +@@ -161,1 +161,1 @@ +-#if !defined(HYPRE_USING_SYCL) ++#if !defined(HYPRE_USING_SYCL) && !defined(HYPRE_USING_HIP) +@@ -187,0 +188,15 @@ ++#if defined(HYPRE_USING_HIP) ++ /* ++ * rocSPARSE's legacy csrilu0 analysis is not functional on the ++ * gfx1100 devices shipped with ROCm 7.2. Use HYPRE's classical ++ * ILU(0) implementation instead. The factorization is performed ++ * on the host only during setup and the resulting LDU factors are ++ * migrated back to the device by hypre_ILUSetupLDUtoVendor. ++ */ ++ if (fill_level == 0 && !(ilu_type % 10)) ++ { ++ hypre_ParCSRMatrixMigrate(Apq, HYPRE_MEMORY_HOST); ++ hypre_ILUSetupILU0(Apq, NULL, NULL, n, n, ++ &parL, &parD, &parU, &parS, &uend); ++ } ++#endif +@@ -214,0 +215,6 @@ ++#if defined(HYPRE_USING_HIP) ++ else if (fill_level == 0 && !(ilu_type % 10)) ++ { ++ hypre_TFree(parD, HYPRE_MEMORY_HOST); ++ } ++#endif diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-mgr-col-lumped-destroy.patch b/scripts/spack_packages/packages/hypre/patches/hypre-mgr-col-lumped-destroy.patch new file mode 100644 index 00000000..6e90feaf --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-mgr-col-lumped-destroy.patch @@ -0,0 +1,13 @@ +diff --git a/src/parcsr_ls/par_mgr_interp.c b/src/parcsr_ls/par_mgr_interp.c +index 649151450b892a7995f762c5a6d87a54588791bc..68de431fa0f6c437575e43c63a9d716c58bb991b 100644 +--- a/src/parcsr_ls/par_mgr_interp.c ++++ b/src/parcsr_ls/par_mgr_interp.c +@@ -2548,6 +2548,8 @@ + block_dim, block_dim, + hypre_DenseBlockMatrixNumNonzeros(B_CF), + hypre_DenseBlockMatrixData(B_CF)); ++ hypre_DenseBlockMatrixDestroy(B_CF); ++ B_CF = NULL; + } + else + { diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-mixedint-export.patch b/scripts/spack_packages/packages/hypre/patches/hypre-mixedint-export.patch new file mode 100644 index 00000000..a6b5554b --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-mixedint-export.patch @@ -0,0 +1,12 @@ +diff --git a/src/parcsr_mv/par_csr_matrix.c b/src/parcsr_mv/par_csr_matrix.c +--- a/src/parcsr_mv/par_csr_matrix.c ++++ b/src/parcsr_mv/par_csr_matrix.c +@@ -2458,1 +2458,1 @@ +- HYPRE_Int contact_send_buf[1]; ++ HYPRE_BigInt contact_send_buf[1]; +@@ -2504,1 +2504,1 @@ +- contact_send_buf[0] = (HYPRE_Int) hypre_ParCSRMatrixLastRowIndex(par_matrix); ++ contact_send_buf[0] = hypre_ParCSRMatrixLastRowIndex(par_matrix); +@@ -2533,1 +2533,1 @@ +- contact_send_buf_starts, sizeof(HYPRE_Int), ++ contact_send_buf_starts, sizeof(HYPRE_BigInt), diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-rcm-capacity.patch b/scripts/spack_packages/packages/hypre/patches/hypre-rcm-capacity.patch new file mode 100644 index 00000000..f28022ba --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-rcm-capacity.patch @@ -0,0 +1,7 @@ +diff --git a/src/parcsr_ls/par_ilu.c b/src/parcsr_ls/par_ilu.c +--- a/src/parcsr_ls/par_ilu.c ++++ b/src/parcsr_ls/par_ilu.c +@@ -2579,1 +2579,2 @@ +- G_capacity = hypre_max((A_nnz * n * n / num_nodes / num_nodes) - num_nodes, 1); ++ G_capacity = hypre_max((HYPRE_Int)((HYPRE_Real) A_nnz * n * n / ++ num_nodes / num_nodes) - num_nodes, 1); diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-tagged-innerprod.patch b/scripts/spack_packages/packages/hypre/patches/hypre-tagged-innerprod.patch new file mode 100644 index 00000000..61c62bb0 --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-tagged-innerprod.patch @@ -0,0 +1,8 @@ +diff --git a/src/seq_mv/vector.c b/src/seq_mv/vector.c +--- a/src/seq_mv/vector.c ++++ b/src/seq_mv/vector.c +@@ -1384,1 +1384,1 @@ +- num_threads * num_tags, ++ num_threads * (num_tags + 1), +@@ -1413,0 +1414,1 @@ ++ hypre_TFree(thread_sums, HYPRE_MEMORY_HOST); diff --git a/scripts/spack_packages/packages/hypre/patches/hypre-umpire-wrapper-lifetime.patch b/scripts/spack_packages/packages/hypre/patches/hypre-umpire-wrapper-lifetime.patch new file mode 100644 index 00000000..93eae9da --- /dev/null +++ b/scripts/spack_packages/packages/hypre/patches/hypre-umpire-wrapper-lifetime.patch @@ -0,0 +1,87 @@ +diff --git a/src/utilities/umpire.c b/src/utilities/umpire.c +--- a/src/utilities/umpire.c ++++ b/src/utilities/umpire.c +@@ -44,0 +45,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -140,0 +141,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -202,0 +203,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -265,0 +266,1 @@ ++ umpire_allocator_delete(&allocator); +@@ -354,43 +358,63 @@ hypre_UmpireFinalize(hypre_Handle *hypre_handle_) + umpire_allocator allocator; + + #if defined(HYPRE_USING_UMPIRE_HOST) +- if (hypre_HandleUmpireOwnHostPool(hypre_handle_)) ++ if (hypre_HandleUmpireHostAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpireHostPool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnHostPool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpireHostPool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpireHostPool(hypre_handle_)); + } + hypre_HandleUmpireHostAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpireHostAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnHostPool(hypre_handle_) = 0; + #endif + + #if defined(HYPRE_USING_UMPIRE_DEVICE) +- if (hypre_HandleUmpireOwnDevicePool(hypre_handle_)) ++ if (hypre_HandleUmpireDeviceAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpireDevicePool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnDevicePool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpireDevicePool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpireDevicePool(hypre_handle_)); + } + hypre_HandleUmpireDeviceAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpireDeviceAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnDevicePool(hypre_handle_) = 0; + #endif + + #if defined(HYPRE_USING_UMPIRE_UM) +- if (hypre_HandleUmpireOwnUMPool(hypre_handle_)) ++ if (hypre_HandleUmpireUMAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpireUMPool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnUMPool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpireUMPool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpireUMPool(hypre_handle_)); + } + hypre_HandleUmpireUMAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpireUMAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnUMPool(hypre_handle_) = 0; + #endif + + #if defined(HYPRE_USING_UMPIRE_PINNED) +- if (hypre_HandleUmpireOwnPinnedPool(hypre_handle_)) ++ if (hypre_HandleUmpirePinnedAllocatorAddress(hypre_handle_)) + { +- allocator = hypre_HandleUmpirePinnedPool(hypre_handle_); +- umpire_allocator_release(&allocator); ++ if (hypre_HandleUmpireOwnPinnedPool(hypre_handle_)) ++ { ++ allocator = hypre_HandleUmpirePinnedPool(hypre_handle_); ++ umpire_allocator_release(&allocator); ++ } ++ umpire_allocator_delete(&hypre_HandleUmpirePinnedPool(hypre_handle_)); + } + hypre_HandleUmpirePinnedAllocatorAddress(hypre_handle_) = NULL; + hypre_HandleUmpirePinnedAllocatorId(hypre_handle_) = 0; ++ hypre_HandleUmpireOwnPinnedPool(hypre_handle_) = 0; + #endif + + return hypre_error_flag; diff --git a/scripts/spack_packages/packages/hypredrive/package.py b/scripts/spack_packages/packages/hypredrive/package.py index e5a68e1b..d1268167 100644 --- a/scripts/spack_packages/packages/hypredrive/package.py +++ b/scripts/spack_packages/packages/hypredrive/package.py @@ -25,10 +25,15 @@ class Hypredrive(CMakePackage, CudaPackage, ROCmPackage): # Keep the complete Git history so CMake can compute the tagged, # distance-aware development version (for example, v0.2.0-2-g). - version("develop", branch="master", get_full_repo=True) + version("develop", branch="master", get_full_repo=True, preferred=True) version("0.2.0", sha256="2fe6c5b2779de41fbd294cb4647c7bbd210ec95934639117e56a790e56c32e41") version("0.1.0", sha256="39db73b75e37457035c64b4c8831abe716bf2f596c4ca79a32293d9bd51ca8d6") + # Match the patches used by the legacy GEOS TPL superbuild. The HIP + # changes are only valid for the HIP backend. + patch("patches/hypredrive-cxx-linker.patch", when="@develop") + patch("patches/hypredrive-hip-rocsparse.patch", when="@develop+rocm") + variant("shared", default=False, description="Build shared libraries") variant("pic", default=False, description="Build position independent code") variant("examples", default=False, description="Build and install example programs") @@ -51,19 +56,19 @@ class Hypredrive(CMakePackage, CudaPackage, ROCmPackage): depends_on("cmake@3.23:", type="build") depends_on("mpi") - depends_on("hypre@2.20.0: +mpi") - depends_on("hypre+shared", when="+shared") - depends_on("hypre@2.21:+pic~shared", when="+pic~shared") - depends_on("hypre+caliper", when="+caliper") - depends_on("hypre+cuda", when="+cuda") - depends_on("hypre+rocm", when="+rocm") - depends_on("hypre@2.24:+sycl", when="+sycl") - depends_on("hypre+superlu-dist", when="+superlu-dist") + depends_on("hypre@develop +mpi") + depends_on("hypre@develop +shared", when="+shared") + depends_on("hypre@develop +pic~shared", when="+pic~shared") + depends_on("hypre@develop +caliper", when="+caliper") + depends_on("hypre@develop +cuda", when="+cuda") + depends_on("hypre@develop +rocm", when="+rocm") + depends_on("hypre@develop +sycl", when="+sycl") + depends_on("hypre@develop +superlu-dist", when="+superlu-dist") for feature in ("fortran", "matlab", "julia", "superlu-dist"): - depends_on("hypre precision=double", when="+{0}".format(feature)) + depends_on("hypre@develop precision=double", when="+{0}".format(feature)) for feature in ("matlab", "julia"): - depends_on("hypre~complex", when="+{0}".format(feature)) + depends_on("hypre@develop~complex", when="+{0}".format(feature)) requires( "%c,cxx=oneapi", @@ -73,7 +78,7 @@ class Hypredrive(CMakePackage, CudaPackage, ROCmPackage): for arch in CudaPackage.cuda_arch_values: depends_on( - "hypre+cuda cuda_arch={0}".format(arch), when="+cuda cuda_arch={0}".format(arch) + "hypre@develop+cuda cuda_arch={0}".format(arch), when="+cuda cuda_arch={0}".format(arch) ) depends_on( "superlu-dist@9.2.1:+cuda cuda_arch={0}".format(arch), @@ -82,7 +87,7 @@ class Hypredrive(CMakePackage, CudaPackage, ROCmPackage): for target in ROCmPackage.amdgpu_targets: depends_on( - "hypre+rocm amdgpu_target={0}".format(target), + "hypre@develop+rocm amdgpu_target={0}".format(target), when="+rocm amdgpu_target={0}".format(target), ) depends_on( diff --git a/scripts/spack_packages/packages/hypredrive/patches/hypredrive-cxx-linker.patch b/scripts/spack_packages/packages/hypredrive/patches/hypredrive-cxx-linker.patch new file mode 100644 index 00000000..8d4adf66 --- /dev/null +++ b/scripts/spack_packages/packages/hypredrive/patches/hypredrive-cxx-linker.patch @@ -0,0 +1,25 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -235,6 +235,12 @@ + $ + INTERFACE + $) ++if(HYPREDRV_HYPRE_USE_OPENMP) ++ find_package(OpenMP REQUIRED COMPONENTS C) ++ if(TARGET OpenMP::OpenMP_C) ++ target_link_libraries(HYPREDRV PUBLIC OpenMP::OpenMP_C) ++ endif() ++endif() + + if(HYPREDRV_INSTRUMENTATION_LINK_FLAGS) + target_link_options(HYPREDRV PUBLIC ${HYPREDRV_INSTRUMENTATION_LINK_FLAGS}) +@@ -419,4 +425,9 @@ + # Define the main executable (hypredrive-cli) + add_executable(hypredrive-cli src/internal/main.c) ++# HYPRE's static dependency graph can contain C++ libraries such as ++# SuperLU_DIST. Select the C++ linker so the C++ runtime is linked as well. ++if(CMAKE_CXX_COMPILER) ++ set_property(TARGET hypredrive-cli PROPERTY LINKER_LANGUAGE CXX) ++endif() + target_link_libraries(hypredrive-cli PRIVATE ${HYPREDRV_INTERNAL_LINK_TARGET}) + hypredrv_add_hypre_build_dependency(hypredrive-cli) diff --git a/scripts/spack_packages/packages/hypredrive/patches/hypredrive-hip-rocsparse.patch b/scripts/spack_packages/packages/hypredrive/patches/hypredrive-hip-rocsparse.patch new file mode 100644 index 00000000..827afc5b --- /dev/null +++ b/scripts/spack_packages/packages/hypredrive/patches/hypredrive-hip-rocsparse.patch @@ -0,0 +1,77 @@ +--- a/src/internal/amg.c ++++ b/src/internal/amg.c +@@ -190,19 +190,29 @@ + *-----------------------------------------------------------------------------*/ + + void + hypredrv_AMGrlxSetDefaultArgs(AMGrlx_args *args) + { ++#ifdef HYPRE_USING_HIP ++ /* Use a device-compatible smoother even when AMG does not coarsen. */ ++ args->type = 18; ++#else + args->type = -1; ++#endif + #ifdef HYPRE_USING_GPU + args->down_type = 18; + args->up_type = 18; + #else + args->down_type = 13; + args->up_type = 14; + #endif ++#ifdef HYPRE_USING_HIP ++ /* Avoid the legacy ROCm CSR triangular-solve path on HIP GPUs. */ ++ args->coarse_type = 18; ++#else + args->coarse_type = 9; ++#endif + args->down_sweeps = -1; + args->up_sweeps = -1; + args->coarse_sweeps = 1; + args->num_sweeps = 1; + args->order = 0; +@@ -1187,7 +1193,12 @@ + #else + HYPRE_BoomerAMGSetILULocalReordering(precon, args->smoother.ilu.reordering); + #endif ++#ifdef HYPRE_USING_HIP ++ /* Keep the classical HIP ILU factors off rocSPARSE csrsv. */ ++ HYPRE_BoomerAMGSetILUTriSolve(precon, 0); ++#else + HYPRE_BoomerAMGSetILUTriSolve(precon, args->smoother.ilu.tri_solve); ++#endif + HYPRE_BoomerAMGSetILULowerJacobiIters(precon, args->smoother.ilu.lower_jac_iters); + HYPRE_BoomerAMGSetILUUpperJacobiIters(precon, args->smoother.ilu.upper_jac_iters); + #endif +--- a/src/internal/ilu.c ++++ b/src/internal/ilu.c +@@ -87,7 +87,12 @@ + HYPRE_ILUSetLocalReordering(precon, args->reordering); + #endif + #if HYPRE_CHECK_MIN_VERSION(22600, 0) ++#ifdef HYPRE_USING_HIP ++ /* HYPRE uses classical host ILU(0) setup on HIP; avoid rocSPARSE csrsv. */ ++ HYPRE_ILUSetTriSolve(precon, 0); ++#else + HYPRE_ILUSetTriSolve(precon, args->tri_solve); ++#endif + HYPRE_ILUSetLowerJacobiIters(precon, args->lower_jac_iters); + HYPRE_ILUSetUpperJacobiIters(precon, args->upper_jac_iters); + #endif +--- a/src/HYPREDRV.c ++++ b/src/HYPREDRV.c +@@ -313,10 +313,13 @@ + #endif + + #if HYPRE_CHECK_MIN_VERSION(22500, 0) + HYPRE_SetSpGemmUseVendor(hypredrv->iargs->general.use_vendor_spgemm); + HYPRE_SetSpMVUseVendor(hypredrv->iargs->general.use_vendor_spmv); ++#ifdef HYPRE_USING_HIP ++ HYPRE_SetSpTransUseVendor(0); ++#endif + #endif + + #ifdef HYPRE_USING_UMPIRE + HYPRE_SetUmpireDevicePoolName("HYPRE_DEVICE"); + HYPRE_SetUmpireUMPoolName("HYPRE_UM"); + HYPRE_SetUmpireHostPoolName("HYPRE_HOST"); diff --git a/scripts/spack_packages/packages/parmetis/package.py b/scripts/spack_packages/packages/parmetis/package.py index 90663c34..1849820d 100644 --- a/scripts/spack_packages/packages/parmetis/package.py +++ b/scripts/spack_packages/packages/parmetis/package.py @@ -7,6 +7,11 @@ class Parmetis(BuiltinParmetis): + # The GEOS superbuild compiles the bundled METIS with 64-bit indices. + # Spack's standalone metis package already performs the equivalent header + # update for +int64; this patch is for ParMETIS's bundled copy. + patch("patches/parmetis-idx64.patch", when="@4.0.3+int64") + # GEOS links SuperLU-DIST as a shared library. Its static ParMETIS # dependency therefore has to contain position-independent code; the # upstream recipe does not propagate dependency cflags into CMake. diff --git a/scripts/spack_packages/packages/parmetis/patches/parmetis-idx64.patch b/scripts/spack_packages/packages/parmetis/patches/parmetis-idx64.patch new file mode 100644 index 00000000..25d21ba1 --- /dev/null +++ b/scripts/spack_packages/packages/parmetis/patches/parmetis-idx64.patch @@ -0,0 +1,13 @@ +diff --git a/metis/include/metis.h b/metis/include/metis.h +index 1cbd4b8..c6537e2 100644 +--- a/metis/include/metis.h ++++ b/metis/include/metis.h +@@ -30,7 +30,7 @@ + GCC does provides these definitions in stdint.h, but it may require some + modifications on other architectures. + --------------------------------------------------------------------------*/ +-#define IDXTYPEWIDTH 32 ++#define IDXTYPEWIDTH 64 + + + /*-------------------------------------------------------------------------- diff --git a/scripts/spack_packages/packages/raja/package.py b/scripts/spack_packages/packages/raja/package.py new file mode 100644 index 00000000..eac972bf --- /dev/null +++ b/scripts/spack_packages/packages/raja/package.py @@ -0,0 +1,12 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * +from spack_repo.builtin.packages.raja.package import Raja as BuiltinRaja + + +class Raja(BuiltinRaja): + # RAJA's CUDA and HIP reducer assignment operators need to preserve the + # ownership of host/device resources used by GEOS kernels. + patch("patches/raja-reducer-assignment.patch", when="@2026.07.0") diff --git a/scripts/spack_packages/packages/raja/patches/raja-reducer-assignment.patch b/scripts/spack_packages/packages/raja/patches/raja-reducer-assignment.patch new file mode 100644 index 00000000..85ad8dc8 --- /dev/null +++ b/scripts/spack_packages/packages/raja/patches/raja-reducer-assignment.patch @@ -0,0 +1,94 @@ +diff --git a/include/RAJA/policy/cuda/reduce.hpp b/include/RAJA/policy/cuda/reduce.hpp +--- a/include/RAJA/policy/cuda/reduce.hpp ++++ b/include/RAJA/policy/cuda/reduce.hpp +@@ -1123,6 +1123,43 @@ + #endif + } + ++ //! assign a reducer while preserving ownership of host and device resources ++ Reduce& operator=(const Reduce& other) ++ { ++ if (this != &other) ++ { ++ // The implicitly generated assignment operator shallow-copies the ++ // reducer state. In particular, it overwrites an owning tally pointer ++ // and leaks the tally allocated by the destination. It also copies the ++ // device-ownership flag, which can make two reducers free the same ++ // device allocation. Match the copy constructor's non-owning semantics ++ // after releasing resources currently owned by the destination. ++ if (parent == this) ++ { ++ delete tally_or_val_ptr.list; ++ } ++ else if (parent == nullptr) ++ { ++ val.teardownForDevice(); ++ } ++ ++ parent = other.parent; ++ tally_or_val_ptr = other.tally_or_val_ptr; ++ val = reduce_data_type(other.val); ++ ++ if (parent && tally_or_val_ptr.list) ++ { ++ if (val.setupForDevice()) ++ { ++ tally_or_val_ptr.val_ptr = val.init_grid_vals( ++ tally_or_val_ptr.list->new_value(currentResource())); ++ parent = nullptr; ++ } ++ } ++ } ++ return *this; ++ } ++ + //! apply reduction upon destruction and cleanup resources owned by this copy + // on device store in pinned buffer on host + RAJA_HOST_DEVICE +diff --git a/include/RAJA/policy/hip/reduce.hpp b/include/RAJA/policy/hip/reduce.hpp +--- a/include/RAJA/policy/hip/reduce.hpp ++++ b/include/RAJA/policy/hip/reduce.hpp +@@ -1116,6 +1116,43 @@ + #endif + } + ++ //! assign a reducer while preserving ownership of host and device resources ++ Reduce& operator=(const Reduce& other) ++ { ++ if (this != &other) ++ { ++ // The implicitly generated assignment operator shallow-copies the ++ // reducer state. In particular, it overwrites an owning tally pointer ++ // and leaks the tally allocated by the destination. It also copies the ++ // device-ownership flag, which can make two reducers free the same ++ // device allocation. Match the copy constructor's non-owning semantics ++ // after releasing resources currently owned by the destination. ++ if (parent == this) ++ { ++ delete tally_or_val_ptr.list; ++ } ++ else if (parent == nullptr) ++ { ++ val.teardownForDevice(); ++ } ++ ++ parent = other.parent; ++ tally_or_val_ptr = other.tally_or_val_ptr; ++ val = reduce_data_type(other.val); ++ ++ if (parent && tally_or_val_ptr.list) ++ { ++ if (val.setupForDevice()) ++ { ++ tally_or_val_ptr.val_ptr = val.init_grid_vals( ++ tally_or_val_ptr.list->new_value(currentResource())); ++ parent = nullptr; ++ } ++ } ++ } ++ return *this; ++ } ++ + //! apply reduction upon destruction and cleanup resources owned by this copy + // on device store in pinned buffer on host + RAJA_HOST_DEVICE