Merge Pull Request #8490 from trilinos/Trilinos/kokkos-promotion-3.3

Automatically Merged using Trilinos Pull Request AutoTester
PR Title: Kokkos promotion to 3.3.01
PR Author: ndellingwood

cmake/std/atdm/contributed/weaver/tweaks/Tweaks.cmake

@@ -9,6 +9,9 @@ ATDM_SET_CACHE(Trilinos_CUDA_SLOTS_PER_GPU 2 CACHE STRING)
 #
 # Disables across multiple builds on 'weaver'
 #
+ATDM_SET_ENABLE(SEACASIoss_structured_cgns_assembly_copy_DISABLE ON)
+ATDM_SET_ENABLE(SEACASIoss_structured_cgns_assembly_copy_fpp_DISABLE ON)
+ATDM_SET_ENABLE(Intrepid2_unit-test_Discretization_Basis_HierarchicalBases_Hierarchical_Basis_Tests_MPI_1_DISABLE ON)
 
 # Disable known failures for SPARC Trilinos configuration (#3632)
 ATDM_SET_ENABLE(PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2_DISABLE ON)

packages/intrepid2/src/Discretization/Integration/Intrepid2_IntegrationToolsDef.hpp

@@ -2086,8 +2086,9 @@ void IntegrationTools<ExecSpaceType>::integrate(Data<Scalar,ExecSpaceType> integ
  haveLaunchedContributionToLeftFamily = true;
  haveLaunchedContributionToRightFamily = true;
 
- std::initializer_list<int> upperBounds {cellDataExtent,leftComponentFieldCount,rightComponentFieldCount}; // separately declared in effort to get around Intel 17.0.1 compiler weirdness.
- auto policy = Kokkos::MDRangePolicy<ExecSpaceType,Kokkos::Rank<3>>({0,0,0}, upperBounds);
+ Kokkos::Array<int,3> upperBounds {cellDataExtent,leftComponentFieldCount,rightComponentFieldCount}; // separately declared in effort to get around Intel 17.0.1 compiler weirdness.
+ Kokkos::Array<int,3> lowerBounds {0,0,0};
+ auto policy = Kokkos::MDRangePolicy<ExecSpaceType,Kokkos::Rank<3>>(lowerBounds, upperBounds);
  // TODO: note that for best performance, especially with Fad types, we should replace this parallel for with a Functor and use hierarchical parallelism
  Kokkos::parallel_for("compute field integrals", policy,
  KOKKOS_LAMBDA (const int &cellDataOrdinal, const int &leftFieldOrdinal, const int &rightFieldOrdinal) {

packages/kokkos-kernels/.gitignore

@@ -0,0 +1,10 @@
+# Standard ignores
+*~
+*.pyc
+\#*#
+.#*
+.*.swp
+.cproject
+.project
+*.o
+TAGS

packages/kokkos-kernels/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Change Log
 
+## [3.3.01](https:/kokkos/kokkos-kernels/tree/3.3.01) (2021-01-18)
+[Full Changelog](https:/kokkos/kokkos-kernels/compare/3.3.00...3.3.01)
+
+**Fixed Bugs:**
+- With CuSparse enabled too many variants of SPMV were instantiated even if not requested. Up to 1GB executable size increase.
+
+## [3.3.00](https:/kokkos/kokkos-kernels/tree/3.3.00) (2020-12-16)
+[Full Changelog](https:/kokkos/kokkos-kernels/compare/3.2.01...3.3.00)
+
+**Implemented enhancements:**
+- Add permanent RCM reordering interface, and a basic serial implementation [\#854](https:/kokkos/kokkos/pull/#854)
+- Half\_t explicit conversions [\#849](https:/kokkos/kokkos/pull/#849)
+- Add batched gemm performance tests [\#838](https:/kokkos/kokkos/pull/#838)
+- Add HIP support to src and perf\_test [\#828](https:/kokkos/kokkos/pull/#828)
+- Factor out coarsening [\#827](https:/kokkos/kokkos/pull/#827)
+- Allow enabling/disabling components at configuration time [\#823](https:/kokkos/kokkos/pull/#823)
+- HIP: CMake work on tests and ETI [\#820](https:/kokkos/kokkos/pull/#820)
+- HIP: KokkosBatched - hip specialization [\#812](https:/kokkos/kokkos/pull/#812)
+- Distance-2 maximal independent set [\#801](https:/kokkos/kokkos/pull/#801)
+- Use batched TRTRI & TRMM for Supernode-sptrsv setup [\#797](https:/kokkos/kokkos/pull/#797)
+- Initial support for half precision [\#794](https:/kokkos/kokkos/pull/#794)
+
+**Fixed bugs:**
+- Fix issue with HIP and Kokkos\_ArithTraits [\#844](https:/kokkos/kokkos/pull/#844)
+- HIP: fixing round of issues on AMD [\#840](https:/kokkos/kokkos/pull/#840)
+- Throw an exception if BLAS GESV is not enabled [\#837](https:/kokkos/kokkos/pull/#837)
+- Fixes -Werror for gcc with c++20 [\#836](https:/kokkos/kokkos/pull/#836)
+- Add fallback condition to use spmv\_native when cuSPARSE does not work [\#834](https:/kokkos/kokkos/pull/#834)
+- Fix install testing refactor for inline builds [\#811](https:/kokkos/kokkos/pull/#811)
+- HIP: fix ArithTraits to support HIP backend [\#809](https:/kokkos/kokkos/pull/#809)
+- cuSPARSE 11: fix spgemm and spmv\_struct\_tunning compilation error [\#804](https:/kokkos/kokkos/pull/#804)
+
+**Incompatibilities:**
+- Remove pre-3.0 deprecated code [\#825](https:/kokkos/kokkos/pull/#825)
+
 ## [3.2.01](https:/kokkos/kokkos-kernels/tree/3.2.01) (2020-11-17)
 [Full Changelog](https:/kokkos/kokkos-kernels/compare/3.2.00...3.2.01)
 
@@ -28,8 +63,8 @@
 - Nightly test failure: spgemm unit tests failing on White \(Power8\) [\#780](https:/kokkos/kokkos-kernels/issues/780)
 - supernodal does not build with UVM enabled [\#633](https:/kokkos/kokkos-kernels/issues/633)
 
-## [3.1.1](https:/kokkos/kokkos-kernels/tree/3.1.1) (2020-05-04)
-[Full Changelog](https:/kokkos/kokkos-kernels/compare/3.1.00...3.1.1)
+## [3.1.01](https:/kokkos/kokkos-kernels/tree/3.1.01) (2020-05-04)
+[Full Changelog](https:/kokkos/kokkos-kernels/compare/3.1.00...3.1.01)
 
 ** Fixed bugs:** 
 

packages/kokkos-kernels/CMakeLists.txt

@@ -24,7 +24,7 @@ IF(NOT KOKKOSKERNELS_HAS_TRILINOS)
  PROJECT(KokkosKernels CXX)
  ENDIF()
  SET(KokkosKernels_VERSION_MAJOR 3)
- SET(KokkosKernels_VERSION_MINOR 2)
+ SET(KokkosKernels_VERSION_MINOR 3)
  SET(KokkosKernels_VERSION_PATCH 1)
 ENDIF()
 
@@ -83,7 +83,8 @@ IF (KokkosKernels_INSTALL_TESTING)
 ELSE()
  # Regular build, not install testing
  # Do all the regular option processing
- IF (NOT KOKKOSKERNELS_HAS_TRILINOS)
+ IF (NOT KOKKOSKERNELS_HAS_TRILINOS AND NOT KOKKOSKERNELS_HAS_PARENT)
+ # This is a standalone build
  FIND_PACKAGE(Kokkos REQUIRED)
  MESSAGE(STATUS "Found Kokkos at ${Kokkos_DIR}")
  KOKKOS_CHECK(OPTIONS CUDA_UVM RETURN_VALUE KOKKOS_ENABLE_CUDA_UVM)
@@ -139,6 +140,15 @@ ELSE()
  BOOL
  "Whether to restrict testing to ETI types. Default: ON"
  )
+
+ KOKKOSKERNELS_ADD_OPTION(
+ ENABLED_COMPONENTS
+ "ALL"
+ STRING
+ "A list of components to enable in testing and building"
+ VALID_ENTRIES BATCHED BLAS GRAPH SPARSE ALL
+ )
+
  # ==================================================================
  # Enable Device Types for ETI (exec- + mem-space)
  # ==================================================================
@@ -159,7 +169,6 @@ ELSE()
  # Enable Layout Types for ETI
  # ==================================================================
  INCLUDE(cmake/kokkoskernels_eti_layouts.cmake)
-
  # ==================================================================
  # Enable Third Party Libraries
  # ==================================================================

packages/kokkos-kernels/cm_generate_makefile.bash

@@ -21,8 +21,18 @@ get_kokkos_device_list() {
  for DEVICE_ in $PARSE_DEVICES_LST
  do 
  UC_DEVICE=$(echo $DEVICE_ | tr "[:lower:]" "[:upper:]")
+ if [ "${UC_DEVICE}" == "CUDA" ]; then
+ WITH_CUDA_BACKEND=ON
+ fi
+ if [ "${UC_DEVICE}" == "HIP" ]; then
+ WITH_HIP_BACKEND=ON
+ fi
  KOKKOS_DEVICE_CMD="-DKokkos_ENABLE_${UC_DEVICE}=ON ${KOKKOS_DEVICE_CMD}"
  done
+ if [ "${WITH_CUDA_BACKEND}" == "ON" ] && [ "${WITH_HIP_BACKEND}" == "ON" ]; then
+ echo "Invalid configuration - Cuda and Hip cannot be simultaneously enabled"
+ exit
+ fi
 }
 
 get_kokkos_arch_list() {
@@ -59,6 +69,24 @@ get_kokkos_cuda_option_list() {
  done
 }
 
+get_kokkos_hip_option_list() {
+ echo parsing KOKKOS_HIP_OPTIONS=$KOKKOS_HIP_OPTIONS
+ KOKKOS_HIP_OPTION_CMD=
+ PARSE_HIP_LST=$(echo $KOKKOS_HIP_OPTIONS | tr "," "\n")
+ for HIP_ in $PARSE_HIP_LST
+ do
+ HIP_OPT_NAME=
+ if [ "${HIP_}" == "rdc" ]; then
+ HIP_OPT_NAME=HIP_RELOCATABLE_DEVICE_CODE
+ else
+ echo "${HIP_} is not a valid hip option..."
+ fi
+ if [ "${HIP_OPT_NAME}" != "" ]; then
+ KOKKOS_HIP_OPTION_CMD="-DKokkos_ENABLE_${HIP_OPT_NAME}=ON ${KOKKOS_HIP_OPTION_CMD}"
+ fi
+ done
+}
+
 get_kokkos_option_list() {
  echo parsing KOKKOS_OPTIONS=$KOKKOS_OPTIONS
  KOKKOS_OPTION_CMD=
@@ -196,15 +224,21 @@ display_help_text() {
  echo "--prefix=/Install/Path: Path to install the KokkosKernels library."
  echo ""
  echo "--with-cuda[=/Path/To/Cuda]: Enable Cuda and set path to Cuda Toolkit."
+ echo "--with-hip[=/Path/To/Hip]: Enable Hip and set path to ROCM Toolkit."
  echo "--with-openmp: Enable OpenMP backend."
  echo "--with-pthread: Enable Pthreads backend."
  echo "--with-serial: Enable Serial backend."
  echo "--with-devices: Explicitly add a set of backends."
  echo ""
  echo "--arch=[OPT]: Set target architectures. Options are:"
- echo " [AMD]"
+ echo " [AMD: CPU]"
  echo " AMDAVX = AMD CPU"
- echo " EPYC = AMD EPYC Zen-Core CPU"
+ echo " ZEN = AMD Zen-Core CPU"
+ echo " ZEN2 = AMD Zen2-Core CPU"
+ echo " [AMD: GPU]"
+ echo " VEGA900 = AMD GPU MI25 GFX900"
+ echo " VEGA906 = AMD GPU MI50/MI60 GFX906"
+ echo " VEGA908 = AMD GPU"
  echo " [ARM]"
  echo " ARMV80 = ARMv8.0 Compatible CPU"
  echo " ARMV81 = ARMv8.1 Compatible CPU"
@@ -264,6 +298,8 @@ display_help_text() {
  echo " "
  echo "--with-cuda-options=[OPT]: Additional options to CUDA:"
  echo " force_uvm, use_ldg, enable_lambda, rdc"
+ echo "--with-hip-options=[OPT]: Additional options to HIP:"
+ echo " rdc"
  echo "--with-scalars=[SCALARS]: Set scalars to be instantiated."
  echo " Options: float, double, complex_float, complex_double"
  echo "--with-ordinals=[ORDINALS]: Set ordinals to be instantiated."
@@ -307,6 +343,10 @@ KOKKOS_MAKEINSTALL_J=4
 
 KERNELS_DEFAULT_ETI_OPTION=""
 
+# For tracking if Cuda and Hip devices are enabled simultaneously
+WITH_CUDA_BACKEND=OFF
+WITH_HIP_BACKEND=OFF
+
 while [[ $# > 0 ]]
 do
  key="$1"
@@ -340,6 +380,19 @@ do
  update_kokkos_devices Cuda
  CUDA_PATH="${key#*=}"
  ;;
+ --with-hip)
+ update_kokkos_devices Hip
+ HIP_PATH_HIPCC=$(command -v hipcc)
+ HIP_PATH=${HIP_PATH_HIPCC%/bin/hipcc}
+ ;;
+ # Catch this before '--with-hip*'
+ --with-hip-options*)
+ KOKKOS_HIP_OPTIONS="${key#*=}"
+ ;;
+ --with-hip*)
+ update_kokkos_devices Hip
+ HIP_PATH="${key#*=}"
+ ;;
  --with-openmp)
  update_kokkos_devices OpenMP
  ;;
@@ -606,6 +659,7 @@ get_kokkos_device_list
 get_kokkos_option_list
 get_kokkos_arch_list
 get_kokkos_cuda_option_list
+get_kokkos_hip_option_list
 
 get_kernels_scalar_list
 get_kernels_ordinals_list
@@ -655,9 +709,9 @@ cd ${KOKKOS_INSTALL_PATH}
 
 # Configure kokkos
 echo ""
-echo cmake $COMPILER_CMD -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
+echo cmake $COMPILER_CMD -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} ${KOKKOS_HIP_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
 echo ""
-cmake $COMPILER_CMD -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS//\"}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
+cmake $COMPILER_CMD -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS//\"}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} ${KOKKOS_HIP_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
 
 # Install kokkos library
 make install -j $KOKKOS_MAKEINSTALL_J

packages/kokkos-kernels/cmake/KokkosKernelsConfig.cmake.in

@@ -11,6 +11,7 @@ find_dependency(Kokkos HINTS @Kokkos_DIR@)
 
 SET(Kokkos_ENABLE_OPENMP @Kokkos_ENABLE_OPENMP@)
 SET(Kokkos_ENABLE_CUDA @Kokkos_ENABLE_CUDA@)
+SET(Kokkos_ENABLE_HIP @Kokkos_ENABLE_HIP@)
 SET(Kokkos_ENABLE_PTHREAD @Kokkos_ENABLE_PTHREAD@) 
 SET(Kokkos_ENABLE_SERIAL @Kokkos_ENABLE_SERIAL@)
 

packages/kokkos-kernels/cmake/KokkosKernels_config.h.in

@@ -34,6 +34,9 @@
 #cmakedefine KOKKOSKERNELS_INST_EXECSPACE_CUDA
 #cmakedefine KOKKOSKERNELS_INST_MEMSPACE_CUDASPACE
 #cmakedefine KOKKOSKERNELS_INST_MEMSPACE_CUDAUVMSPACE
+/* Whether to build kernels for execution space Kokkos::Experimental::HIP */
+#cmakedefine KOKKOSKERNELS_INST_EXECSPACE_HIP
+#cmakedefine KOKKOSKERNELS_INST_MEMSPACE_HIPSPACE
 /* Whether to build kernels for execution space Kokkos::OpenMP */
 #cmakedefine KOKKOSKERNELS_INST_EXECSPACE_OPENMP
 /* Whether to build kernels for execution space Kokkos::Threads */