Update LandIce summit scripts

doc/summit/LandIce/do-cmake-trilinos-summit-cuda-gcc → doc/LandIce/summit/do-cmake-trilinos-summit-cuda-gcc

@@ -40,8 +40,8 @@ cmake \
     \
     -DTPL_ENABLE_Netcdf:BOOL=ON \
     -DTPL_Netcdf_PARALLEL:BOOL=ON \
-    -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/include \
-    -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/lib \
+    -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/include \
+    -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/lib \
     \
     -DTPL_ENABLE_Pnetcdf:STRING=ON \
     -DTPL_Pnetcdf_INCLUDE_DIRS:FILEPATH=${OLCF_PARALLEL_NETCDF_ROOT}/include \
@@ -70,6 +70,7 @@ cmake \
     -DKokkos_ARCH_POWER9=ON \
     -DKokkos_ARCH_VOLTA70=ON \
     -DTPL_ENABLE_Thrust:BOOL=ON \
+    -DThrust_INCLUDE_DIRS:FILEPATH=${OLCF_CUDA_ROOT}/include \
     -DTPL_ENABLE_CUDA:BOOL=ON \
     -DTPL_ENABLE_CUSPARSE:BOOL=ON \
     \
@@ -81,7 +82,6 @@ cmake \
     -DPhalanx_KOKKOS_DEVICE_TYPE:STRING=CUDA \
     -DSacado_ENABLE_COMPLEX:BOOL=OFF \
     -DTeuchos_ENABLE_COMPLEX:BOOL=OFF \
-    -DTpetra_ENABLE_Kokkos_Refactor:BOOL=ON \
     -DTpetra_ASSUME_CUDA_AWARE_MPI:BOOL=ON \
     -DTpetra_INST_INT_LONG_LONG:BOOL=ON \
     -DTpetra_INST_INT_INT:BOOL=OFF \
@@ -144,8 +144,8 @@ cmake \
     -D Kokkos_ENABLE_LIBDL:BOOL=ON \
     -D Trilinos_ENABLE_PanzerDofMgr:BOOL=ON \
     -D Trilinos_ENABLE_ShyLU_DDFROSch:BOOL=ON \
-    -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \
-    -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \
+    -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \
+    -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \
     \
     ${TRILINOS_DIR}
 

doc/summit/LandIce/do-cmake-trilinos-summit-serial-gcc → doc/LandIce/summit/do-cmake-trilinos-summit-serial-gcc

@@ -39,8 +39,8 @@ cmake \
     \
     -DTPL_ENABLE_Netcdf:BOOL=ON \
     -DTPL_Netcdf_PARALLEL:BOOL=ON \
-    -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/include \
-    -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/lib \
+    -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/include \
+    -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/lib \
     \
     -DTPL_ENABLE_Pnetcdf:STRING=ON \
     -DTPL_Pnetcdf_INCLUDE_DIRS:FILEPATH=${OLCF_PARALLEL_NETCDF_ROOT}/include \
@@ -78,7 +78,6 @@ cmake \
     -DPhalanx_KOKKOS_DEVICE_TYPE:STRING=SERIAL \
     -DSacado_ENABLE_COMPLEX:BOOL=OFF \
     -DTeuchos_ENABLE_COMPLEX:BOOL=OFF \
-    -DTpetra_ENABLE_Kokkos_Refactor:BOOL=ON \
     -DTpetra_ASSUME_CUDA_AWARE_MPI:BOOL=OFF \
     -DTpetra_INST_INT_LONG_LONG:BOOL=ON \
     -DTpetra_INST_INT_INT:BOOL=OFF \
@@ -141,8 +140,8 @@ cmake \
     -D Kokkos_ENABLE_LIBDL:BOOL=ON \
     -D Trilinos_ENABLE_PanzerDofMgr:BOOL=ON \
     -D Trilinos_ENABLE_ShyLU_DDFROSch:BOOL=ON \
-    -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \
-    -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \
+    -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \
+    -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \
     \
     ${TRILINOS_DIR}
 

doc/summit/LandIce/nvcc_wrapper70 → doc/LandIce/summit/nvcc_wrapper70

@@ -96,10 +96,10 @@ replace_pragma_ident=0
 first_xcompiler_arg=1
 
 # Allow for setting temp dir without setting TMPDIR in parent (see https://docs.olcf.ornl.gov/systems/summit_user_guide.html#setting-tmpdir-causes-jsm-jsrun-errors-job-state-flip-flop)
-if [[ ! -z ${NVCC_WRAPPER_TMPDIR+x} ]]; then
+if [[ -z ${NVCC_WRAPPER_TMPDIR+x} ]]; then
   temp_dir=${TMPDIR:-/tmp}
 else
-  temp_dir=${NVCC_WRAPPER_TMPDIR+x}
+  temp_dir=${NVCC_WRAPPER_TMPDIR}
 fi
 
 # optimization flag added as a command-line argument
@@ -152,8 +152,13 @@ do
    # Ensure we only have one optimization flag because NVCC doesn't allow multiple
   -O*)
     if [ -n "$optimization_flag" ]; then
-       echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the last is used because nvcc can only accept a single optimization setting."
-       shared_args=${shared_args/ $optimization_flag/}
+        if [ "$1" = "$optimization_flag" ]; then
+            # Silently consume duplicates of the same argument
+            shift
+            continue
+        fi
+        echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the last is used because nvcc can only accept a single optimization setting."
+        shared_args=${shared_args/ $optimization_flag/}
     fi
     if [ "$1" = "-O" ]; then
       optimization_flag="-O2"
@@ -222,21 +227,92 @@ do
     fi
     ;;
   #Handle known nvcc args
-  --dryrun|--verbose|--keep|--keep-dir*|-G|-lineinfo|-expt-extended-lambda|-expt-relaxed-constexpr|--resource-usage|-Xptxas*|--fmad=*|--use_fast_math|--Wext-lambda-captures-this|-Wext-lambda-captures-this)
+  --dryrun|--verbose|--keep|--source-in-ptx|-src-in-ptx|--keep-dir*|-G|-lineinfo|-expt-extended-lambda|-expt-relaxed-constexpr|--resource-usage|--fmad=*|--use_fast_math|--Wext-lambda-captures-this|-Wext-lambda-captures-this)
     cuda_args="$cuda_args $1"
     ;;
   #Handle more known nvcc args
-  --expt-extended-lambda|--expt-relaxed-constexpr|--Wno-deprecated-gpu-targets|-Wno-deprecated-gpu-targets)
+  --expt-extended-lambda|--expt-relaxed-constexpr|--Wno-deprecated-gpu-targets|-Wno-deprecated-gpu-targets|-allow-unsupported-compiler|--allow-unsupported-compiler)
     cuda_args="$cuda_args $1"
     ;;
   #Handle known nvcc args that have an argument
-  -maxrregcount=*|--maxrregcount=*)
+  -maxrregcount=*|--maxrregcount=*|-time=*|-Xptxas=*)
     cuda_args="$cuda_args $1"
     ;;
-  -maxrregcount|--default-stream|-Xnvlink|--fmad|-cudart|--cudart|-include)
+  -maxrregcount|--default-stream|-Xnvlink|--fmad|-cudart|--cudart|-include|-time|-Xptxas)
     cuda_args="$cuda_args $1 $2"
     shift
     ;;
+  # Handle Werror. Note, we must differentiate between the ones going to nvcc and the host compiler
+  # --Werror kind,... OR --Werror=kind,... <- always to nvcc
+  --Werror)
+    cuda_args="$cuda_args $1 $2"
+    shift
+    ;;
+  --Werror=*)
+    cuda_args="$cuda_args $1"
+    ;;
+  # -Werror kind,... where kind is one of {all-warnings, cross-execution-space-call, reorder, default-stream-launch, missing-launch-bounds, ext-lambda-captures-this, deprecated-declarations} <- goes to nvcc
+  # -Werror not followed by any kind as mentioned above goes to host compiler without any arguments
+  -Werror)
+    if [ $# -gt 1 ]; then
+      IFS="," read -r -a kinds <<< "$2"
+      first_kind=${kinds[0]}
+      # check if the first kind is one of the allowed ones, then this must be an nvcc list so put all of them to the cuda compiler
+      case $first_kind in
+      all-warnings|cross-execution-space-call|reorder|default-stream-launch|missing-launch-bounds|ext-lambda-captures-this|deprecated-declarations)
+        cuda_args="$cuda_args $1 $2"
+        shift
+        ;;
+      *)
+        if [ $first_xcompiler_arg -eq 1 ]; then
+          xcompiler_args="$1"
+          first_xcompiler_arg=0
+        else
+          xcompiler_args="$xcompiler_args,$1"
+        fi
+        ;;
+      esac
+    fi
+    ;;
+  # -Werror=kind,... will be split into two parts, those kinds that belong to nvcc (see above) go there, while all others go towards the host compiler
+  -Werror=*)
+    kinds_str="${1:8}" # strip -Werror=
+    IFS="," read -r -a kinds <<< ${kinds_str}
+    first_werror_cuda=1
+    first_werror_host=1
+    xcompiler_args_werror=
+    # loop over all kinds that are sparated via ','
+    for kind in "${kinds[@]}"
+    do
+      case ${kind} in
+      all-warnings|cross-execution-space-call|reorder|default-stream-launch|missing-launch-bounds|ext-lambda-captures-this|deprecated-declarations)
+        if [ $first_werror_cuda -ne 0 ]; then
+          cuda_args="$cuda_args -Werror="
+          first_werror_cuda=0
+        else
+          cuda_args="$cuda_args,"
+        fi
+        cuda_args="$cuda_args$kind"
+        ;;
+      *)
+        if [ $first_werror_host -eq 0 ]; then
+            xcompiler_args_werror="${xcompiler_args_werror},"
+        fi
+        first_werror_host=0
+        xcompiler_args_werror="$xcompiler_args_werror-Werror=$kind"
+        ;;
+      esac
+    done
+    if [ $first_werror_host -eq 0 ]; then
+      if [ $first_xcompiler_arg -eq 1 ]; then
+        xcompiler_args="$xcompiler_args_werror"
+        first_xcompiler_arg=0
+      else
+        xcompiler_args="$xcompiler_args,$xcompiler_args_werror"
+      fi
+    fi
+    ;;
+  # End of Werror handling
   #Handle unsupported standard flags
   --std=c++1y|-std=c++1y|--std=gnu++1y|-std=gnu++1y|--std=c++1z|-std=c++1z|--std=gnu++1z|-std=gnu++1z|--std=c++2a|-std=c++2a)
     fallback_std_flag="-std=c++14"
@@ -305,7 +381,7 @@ do
   -std=c++98|--std=c++98)
     ;;
   #strip of pedantic because it produces endless warnings about #LINE added by the preprocessor
-  -pedantic|-Wpedantic|-ansi)
+  -pedantic|-pedantic-errors|-Wpedantic|-ansi)
     ;;
   #strip of -Woverloaded-virtual to avoid "cc1: warning: command line option ‘-Woverloaded-virtual’ is valid for C++/ObjC++ but not for C"
   -Woverloaded-virtual)
@@ -552,14 +628,14 @@ if [ $host_only -eq 1 ]; then
   $host_command
 elif [ -n "$nvcc_depfile_command" ]; then
   if [ "$NVCC_WRAPPER_SHOW_COMMANDS_BEING_RUN" == "1" ] ; then
-    echo "$nvcc_command && $nvcc_depfile_command"
+    echo "TMPDIR=${temp_dir} $nvcc_command && TMPDIR=${temp_dir} $nvcc_depfile_command"
   fi
-  $nvcc_command && $nvcc_depfile_command
+  TMPDIR=${temp_dir} $nvcc_command && TMPDIR=${temp_dir} $nvcc_depfile_command
 else
   if [ "$NVCC_WRAPPER_SHOW_COMMANDS_BEING_RUN" == "1" ] ; then
-    echo "$nvcc_command"
+    echo "TMPDIR=${temp_dir} $nvcc_command"
   fi
-  $nvcc_command
+  TMPDIR=${temp_dir} $nvcc_command
 fi
 error_code=$?
 

doc/summit/LandIce/summit_gcc_modules.sh → doc/LandIce/summit/summit_gcc_modules.sh

@@ -6,9 +6,11 @@ module unload xl
 module load gcc/9.1.0
 module load spectrum-mpi/10.4.0.3-20210112
 module load git cmake
-module load boost/1.76.0
+#module load boost/1.76.0
+module load boost/1.77.0
 module load hdf5/1.10.7
-module load netcdf-c/4.8.0
+#module load netcdf-c/4.8.0
+module load netcdf-c/4.8.1
 module load netcdf-fortran/4.4.5
 module load parallel-netcdf/1.12.2
 module load netlib-lapack/3.9.1