Skip to content

Building Albany on Titan

Jump to bottom
Irina K. Tezaur edited this page Oct 13, 2016 · 3 revisions

Detailed build and run instructions with Serial, OpenMP and CUDA node (10/13/2016)

Please refer to SAND report written by Jerry Watkins, which can be found here.

Using GCC compiler + OpenMP/Serial node

  1. Building Trilinos:

1.1. unload/load next:

module unload cmake
module unload cray-hdf5
module unload cray-hdf5-parallel
module unload netcdf
module unload python
module unload cray-shmem
module unload cray-mpich cray-mpich2
module unload netcdf-hdf5parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cray-hdf5-parallel
module unload boost
module unload dynamic-link
module unload pgi
module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi

module load modules
module load cmake/2.8.11.2
module load PrgEnv-gnu
module unload gcc
module load cray-shmem
module load cray-mpich
module load gcc
module load python
module load boost/1.57.0
module load dynamic-link
module load cray-netcdf-hdf5parallel boost
module load cray-hdf5-parallel
module list

1.2 configure and build Trilinos. Configure script example:

 rm -f CMakeCache.txt
 rm -rf CMakeLists

 export TRILINOS_HOME=/ccs/home/ipdemes/TrilinosDir/Trilinos_github/Trilinos
 BOOSTDIR=/ccs/home/ikalash/Install/boost_1_55_0
 SUPERLUDIR=/ccs/home/ikalash/Install/SuperLU_4.3_2
 HWLOC_PATH=/ccs/home/ipdemes/install/hwloc_static
 BUILD_DIR=`pwd`

 EXTRA_ARGS=$@

 rm -rf CMakeFiles CMakeCache.txt

 cmake -D CMAKE_INSTALL_PREFIX:PATH=$BUILD_DIR/install \
  -D Boost_INCLUDE_DIRS:FILEPATH=$BOOSTDIR/include \
  -D Netcdf_LIBRARY_DIRS:FILEPATH="${NETCDF_DIR}/lib" \
  -D Netcdf_INCLUDE_DIRS:FILEPATH="${NETCDF_DIR}/include" \
  -D BoostLib_LIBRARY_DIRS:FILEPATH=$BOOSTDIR/lib \
  -D BoostLib_INCLUDE_DIRS:FILEPATH=$BOOSTDIR/include \
  -D TPL_ENABLE_BoostAlbLib:BOOL=ON \
  -D BoostAlbLib_INCLUDE_DIRS:FILEPATH=$BOOSTDIR/include \
  -D BoostAlbLib_LIBRARY_DIRS:FILEPATH=$BOOSTDIR/lib \
  -D CMAKE_BUILD_TYPE:STRING=RELEASE \
  -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \
  -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \
  -D Trilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=OFF \
 \
  -D Trilinos_ENABLE_Fortran:BOOL=ON \
 \
  -D TPL_ENABLE_SuperLU:BOOL=ON \
  -D SuperLU_INCLUDE_DIRS:PATH=$SUPERLUDIR/SRC \
  -D TPL_SuperLU_LIBRARIES:PATH=$SUPERLUDIR/lib/libsuperlu_4.3.a \
 \
  -D Trilinos_ASSERT_MISSING_PACKAGES=OFF \
  -D Trilinos_ENABLE_Teuchos:BOOL=ON \
  -D HAVE_TEUCHOS_COMM_TIMERS=ON \
  -D Trilinos_ENABLE_Shards:BOOL=ON \
  -D Trilinos_ENABLE_Sacado:BOOL=ON \
  -D Trilinos_ENABLE_Epetra:BOOL=ON \
  -D Trilinos_ENABLE_Tpetra:BOOL=ON \
  -D Trilinos_ENABLE_EpetraExt:BOOL=ON \
  -D Trilinos_ENABLE_Ifpack:BOOL=ON \
  -D Trilinos_ENABLE_Ifpack2:BOOL=ON \
  -D Trilinos_ENABLE_AztecOO:BOOL=ON \
  -D Trilinos_ENABLE_Amesos:BOOL=ON \
  -D Trilinos_ENABLE_Amesos2:BOOL=ON \
  -D Trilinos_ENABLE_Anasazi:BOOL=ON \
  -D Trilinos_ENABLE_Belos:BOOL=ON \
  -D Trilinos_ENABLE_Phalanx:BOOL=ON \
  -D Phalanx_ENABLE_COMPILETIME_ARRAY_CHECK:BOOL=ON \
  -D Phalanx_KOKKOS_DEVICE_TYPE:STRING="OPENMP" \
  -D Phalanx_INDEX_SIZE_TYPE="UINT" \
  -D Tpetra_ENABLE_Kokkos_Refactor:BOOL=ON \
  -D KokkosClassic_DefaultNode:STRING="Kokkos::Compat::KokkosOpenMPWrapperNode" \
  -D Trilinos_ENABLE_TeuchosKokkosCompat:BOOL=ON \
  -D Trilinos_ENABLE_TpetraKernels:BOOL=ON \
  -D Trilinos_ENABLE_TeuchosKokkosComm:BOOL=ON \
  -D Trilinos_ENABLE_Kokkos:BOOL=ON \
  -D Trilinos_ENABLE_KokkosCore:BOOL=ON \
  -D Trilinos_ENABLE_KokkosClassic:BOOL=ON \
  -D Trilinos_ENABLE_KokkosContainers:BOOL=ON \
  -D Trilinos_ENABLE_KokkosLinAlg:BOOL=ON \
  -D Trilinos_ENABLE_KokkosCompat:BOOL=ON \
  -D Kokkos_ENABLE_OpenMP:BOOL=ON \
  -D Trilinos_ENABLE_OpenMP:BOOL=ON \
  -D TPL_ENABLE_OpenMP:BOOL=ON \
  -D TPL_HWLOC_LIBRARIES:PATHNAME="${HWLOC_PATH}/lib/libhwloc.a" \
  -D TPL_HWLOC_INCLUDE_DIRS:PATHNAME="${HWLOC_PATH}/include" \
  -D HWLOC_LIBRARY_DIRS="${HWLOC_PATH}/lib" \
  -D TPL_HWLOC_LIBRARIES="${HWLOC_PATH}/lib/libhwloc.a" \
  -D TPL_ENABLE_HWLOC:STRING=ON \
  -D Trilinos_ENABLE_Intrepid:BOOL=ON \
  -D HAVE_INTREPID_KOKKOSCORE:BOOL=ON \
  -D Trilinos_ENABLE_ML:BOOL=ON \
  -D Trilinos_ENABLE_MueLu:BOOL=ON \
  -D Trilinos_ENABLE_NOX:BOOL=ON \
  -D Trilinos_ENABLE_Stratimikos:BOOL=ON \
  -D Trilinos_ENABLE_Thyra:BOOL=ON \
  -D Trilinos_ENABLE_ThyraTpetraAdapters:BOOL=ON \
  -D Trilinos_ENABLE_TrilinosCouplings:BOOL=ON \
  -D Trilinos_ENABLE_Rythmos:BOOL=ON \
  -D Trilinos_ENABLE_OptiPack:BOOL=ON \
  -D Trilinos_ENABLE_GlobiPack:BOOL=ON \
  -D Trilinos_ENABLE_Stokhos:BOOL=ON \
  -D Trilinos_ENABLE_Isorropia:BOOL=ON\
  -D Trilinos_ENABLE_Piro:BOOL=ON \
  -D Trilinos_ENABLE_STKIO:BOOL=ON \
  -D Trilinos_ENABLE_STKMesh:BOOL=ON \
  -D Trilinos_ENABLE_SEACASExodus:BOOL=ON \
  -D Trilinos_ENABLE_Teko:BOOL=ON \
  -D TPL_FIND_SHARED_LIBS:BOOL=OFF \
 \
  -D Trilinos_ENABLE_TriKota:BOOL=OFF \
  -D TriKota_ENABLE_DakotaCMake:BOOL=OFF \
  -D DAKOTA_ENABLE_TESTS:BOOL=OFF \
  -D Boost_LIBRARY_DIRS:FILEPATH=$BOOSTDIR/lib \
 \
  -D Trilinos_ENABLE_SEACASIoss:BOOL=ON \
  -D Trilinos_ENABLE_ThreadPool:STRING="ON" \
  -D Trilinos_ENABLE_Pamgen:BOOL=ON \
  -D TPL_ENABLE_Netcdf:BOOL=ON \
  -D TPL_Netcdf_INCLUDE_DIRS:PATH="${NETCDF_DIR}/include" \
  -D Netcdf_LIBRARY_DIRS:PATH="${NETCDF_DIR}/lib" \
 \
  -D Trilinos_ENABLE_Mesquite:BOOL=OFF\
  -D Trilinos_ENABLE_Zoltan:BOOL=ON \
  -D Trilinos_ENABLE_Zoltan2:BOOL=ON\
  -D Zoltan_ENABLE_ULONG_IDS:BOOL=ON \
  -D ZOLTAN_BUILD_ZFDRIVE:BOOL=OFF \
  -D Zoltan2_ENABLE_Experimental:BOOL=ON\
  -D Trilinos_ENABLE_FEI:BOOL=OFF\
 \
  -D Trilinos_ENABLE_TESTS:BOOL=OFF \
  -D Piro_ENABLE_TESTS:BOOL=OFF \
  -D Trilinos_ENABLE_EXAMPLES:BOOL=OFF \
  -D TPL_ENABLE_MPI:BOOL=ON \
  -D TPL_ENABLE_Boost:BOOL=ON \
  -D TPL_ENABLE_BoostLib:BOOL=ON \
 \
  -D Anasazi_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D AztecOO_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Belos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Ifpack_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D NOX_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Phalanx_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Rythmos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Stokhos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Stratimikos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Thyra_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Trilinos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
 \
  -D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF \
  -D Trilinos_VERBOSE_CONFIGURE:BOOL=OFF \
  -D CMAKE_CXX_FLAGS:STRING="-O2" \
  -D CMAKE_Fortran_FLAGS:STRING="-O2" \
  -D Trilinos_ENABLE_Export_Makefiles:BOOL=ON \
 \
  -D CMAKE_C_COMPILER:FILEPATH="cc" \
  -D CMAKE_CXX_COMPILER:FILEPATH="CC" \
  -D CMAKE_Fortran_COMPILER:FILEPATH="ftn" \
  -D CMAKE_SHARED_LINKER_FLAGS="/ccs/home/ipdemes/TrilinosDir/BuildTrilinos_Albany_OpenMP/install/lib/cmake/Trilinos/../../../lib /usr/lib64/libxml*.a /usr/lib64/libpciaccess.a  -lnuma" \
  -D TPL_BLAS_LIBRARIES:STRING="sci_gnu_mp" \
  -D TPL_LAPACK_LIBRARIES:STRING="sci_gnu_mp" \
  -DTPL_ENABLE_Pthread:BOOL=OFF \
  -DTPL_ENABLE_BinUtils:BOOL=OFF \
 \
  -D MPI_EXEC:FILEPATH=aprun \
  -D MPI_EXEC_MAX_NUMPROCS:STRING=4 \
  -D MPI_EXEC_NUMPROCS_FLAG:STRING=-n \
 \
 $EXTRA_ARGS \
 ${TRILINOS_HOME}

  1. Building Albany:

    2.1 use the same module script as for Trilinos

    2.2 use your usual Albany script to configure Albany (please see my script "do-configure" attached)

    2.3 building Albany is a little tricky, so here is my step-by-step instruction:

    make -j 4 Albany

    you'll get a linking error. To fix this do next:

    cd src/

    add " /usr/lib64/libxml*.a /usr/lib64/libpciaccess.a -static-libstdc++ -static-libgcc" to the end of the CMakeFiles/Albany.dir/link.txt -lnuma vi CMakeFiles/Albany.dir/link.txt

    copy the linking script from the CMakeFiles/Albany.dir/link.txt and paste into the terminal

    You need to repeat this step for every Albany executable.

Compiling Trilinos for CUDA on Titan:

  1. load modules: module unload cmake module unload cray-hdf5 module unload cray-hdf5-parallel module unload netcdf module unload python module unload cray-shmem module unload cray-mpich cray-mpich2 module unload netcdf-hdf5parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cray-hdf5-parallel module unload boost module unload dynamic-link module unload pgi module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi module unload cray-libsci cudatoolkit

    module load modules module load cmake/2.8.11.2 module load PrgEnv-gnu
    module unload gcc module load cray-shmem module load cray-mpich module swap cray-libsci cray-libsci/13.1.0 module swap cray-mpich cray-mpich/7.2.4 module load gcc/4.8.2 module load python module load boost/1.57.0 module load dynamic-link module load cray-netcdf-hdf5parallel boost module load cudatoolkit export CRAY_CPU_TARGET=x86-64 export CRAY_CUDA_PROXY=1 export CRAY_CPU_TARGET=istanbul module list

  2. Please have next example as a reference for the Trilinos configure script for CUDA on Titan:

    TRILINOS_HOME=/ccs/home/ipdemes/TrilinosDir/Trilinos_github/Trilinos TRILINSTALLDIR=/ccs/home/ipdemes/TrilinosDir/BuildTrilinos_Albany_CUDA/install HDF5DIR=/opt/cray/hdf5-parallel/1.8.14/gnu/4.8 BOOSTDIR=/sw/xk6/boost/1.57.0/cle5.2_gnu4.8.2/ SUPERLUDIR=/ccs/home/ikalash/Install/SuperLU_4.3_2 HWLOC_PATH=/ccs/home/ipdemes/install/hwloc_static

    export BOOST_ROOT=$BOOST_DIR

    EXTRA_ARGS=$@

    rm -rf CMakeFiles CMakeCache.txt

    cmake
    -D Trilinos_DISABLE_ENABLED_FORWARD_DEP_PACKAGES=ON
    -D CMAKE_INSTALL_PREFIX:PATH=${TRILINSTALLDIR}
    -D CMAKE_BUILD_TYPE:STRING=RELEASE \
    -D TPL_ENABLE_MPI:BOOL=ON
    -D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF
    -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF
    -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING=""
    -D Teuchos_ENABLE_LONG_LONG_INT:BOOL=OFF

    -D TPL_FIND_SHARED_LIBS:BOOL=OFF
    -D Trilinos_ENABLE_Teuchos:BOOL=ON
    -D Trilinos_ENABLE_Shards:BOOL=ON
    -D Trilinos_ENABLE_Sacado:BOOL=ON
    -D Trilinos_ENABLE_Epetra:BOOL=ON
    -D Trilinos_ENABLE_EpetraExt:BOOL=ON
    -D Trilinos_ENABLE_Ifpack:BOOL=ON
    -D Trilinos_ENABLE_AztecOO:BOOL=ON
    -D Trilinos_ENABLE_Amesos:BOOL=ON
    -D Trilinos_ENABLE_Anasazi:BOOL=ON
    -D Trilinos_ENABLE_Belos:BOOL=ON
    -D Trilinos_ENABLE_ML:BOOL=ON
    -D Trilinos_ENABLE_Phalanx:BOOL=ON
    -D Phalanx_ENABLE_EXAMPLES:BOOL=OFF
    -D Phalanx_ENABLE_TESTS:BOOL=OFF
    -D Trilinos_ENABLE_Intrepid:BOOL=ON
    -D Trilinos_ENABLE_NOX:BOOL=ON
    -D Trilinos_ENABLE_Stratimikos:BOOL=ON
    -D Trilinos_ENABLE_Thyra:BOOL=ON
    -D Trilinos_ENABLE_Rythmos:BOOL=ON
    -D Trilinos_ENABLE_MOOCHO:BOOL=ON
    -D Trilinos_ENABLE_Stokhos:BOOL=ON
    -D Trilinos_ENABLE_Piro:BOOL=ON
    -D Trilinos_ENABLE_Teko:BOOL=OFF

    -D Trilinos_ENABLE_STKIO:BOOL=ON
    -D Trilinos_ENABLE_STKMesh:BOOL=ON
    -D TPL_ENABLE_Boost:BOOL=ON
    -D Boost_INCLUDE_DIRS:FILEPATH="$BOOSTDIR/include"
    -D Boost_LIBRARY_DIRS:FILEPATH="$BOOSTDIR/lib"
    -D TPL_ENABLE_BoostLib:BOOL=ON
    -D BoostLib_INCLUDE_DIRS:FILEPATH="$BOOSTDIR/include"
    -D BoostLib_LIBRARY_DIRS:FILEPATH="$BOOSTDIR/lib"
    -D CMAKE_C_COMPILER:FILEPATH="cc"
    -D CMAKE_CXX_COMPILER=/ccs/home/ipdemes/TrilinosDir/Albany_github/Albany/doc/nvcc_wrapper_Glen
    -D CMAKE_Fortran_COMPILER:FILEPATH="ftn"
    -D TPL_BLAS_LIBRARIES:STRING="sci_gnu_mp"
    -D TPL_LAPACK_LIBRARIES:STRING="sci_gnu_mp"
    -D CMAKE_CXX_FLAGS="-D_LZCNTINTRIN_H_INCLUDED"
    -D Trilinos_CXX11_FLAGS="-std=c++11"
    -D CMAKE_SHARED_LINKER_FLAGS="/ccs/home/ipdemes/TrilinosDir/BuildTrilinos_Albany_CUDA/install/lib/cmake/Trilinos /../../../lib /usr/lib64/libxml*.a /usr/lib64/libpciaccess.a"

    -D Trilinos_ENABLE_SEACASIoss:BOOL=ON
    -D TPL_ENABLE_Netcdf:BOOL=ON
    -D Netcdf_INCLUDE_DIRS:PATH="${NETCDF_DIR}/include"
    -D Netcdf_LIBRARY_DIRS:PATH="${NETCDF_DIR}/lib"
    -D TPL_ENABLE_HDF5:BOOL=ON
    -D HDF5_INCLUDE_DIRS:PATH="$HDF5DIR/include"
    -D HDF5_LIBRARY_DIRS:PATH="$HDF5DIR/lib"

    -D Trilinos_ENABLE_Tpetra:BOOL=ON
    -D Trilinos_ENABLE_Kokkos:BOOL=ON
    -D Trilinos_ENABLE_Ifpack:BOOL=ON
    -D Trilinos_ENABLE_Amesos2:BOOL=ON
    -D Trilinos_ENABLE_Zoltan2:BOOL=ON
    -D Trilinos_ENABLE_Ifpack2:BOOL=ON
    -D Trilinos_ENABLE_ML:BOOL=ON
    -D Trilinos_ENABLE_MueLu:BOOL=ON
    -D Amesos2_ENABLE_KLU2:BOOL=ON

    -D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON
    -D Tpetra_INST_INT_LONG_LONG:BOOL=OFF
    -D Tpetra_INST_INT_INT:BOOL=ON
    -D Tpetra_INST_DOUBLE:BOOL=ON
    -D Tpetra_INST_FLOAT:BOOL=OFF
    -D Tpetra_INST_COMPLEX_FLOAT:BOOL=OFF
    -D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF
    -D Tpetra_INST_INT_LONG:BOOL=OFF
    -D Tpetra_INST_INT_UNSIGNED:BOOL=OFF
    -D Tpetra_INST_SERIAL=ON

    -D Trilinos_ENABLE_Kokkos:BOOL=ON
    -D Trilinos_ENABLE_KokkosCore:BOOL=ON
    -D Phalanx_KOKKOS_DEVICE_TYPE:STRING="CUDA"
    -D Phalanx_INDEX_SIZE_TYPE:STRING="INT"
    -D Phalanx_SHOW_DEPRECATED_WARNINGS:BOOL=OFF
    -D Kokkos_ENABLE_Serial:BOOL=ON
    -D Kokkos_ENABLE_OpenMP:BOOL=OFF
    -D Kokkos_ENABLE_Pthread:BOOL=OFF
    -D Trilinos_ENABLE_OpenMP:BOOL=OFF
    -D Kokkos_ENABLE_Cuda:BOOL=ON
    -D Kokkos_ENABLE_Cuda_UVM=ON
    -D TPL_ENABLE_CUDA:BOOL=ON
    -D TPL_ENABLE_CUSPARSE:BOOL=ON
    -D HAVE_INTREPID_KOKKOSCORE:BOOL=ON

    $EXTRA_ARGS
    ${TRILINOS_HOME}

  3. other steps look are similar to the ones for OpenMP

Docker Image

SPACK Package

ToDo List

Albany FAQs

Capabilities

Sub-Projects

Compiling

Nightly Tests

Code HowTo

User Group Meetings

PAALS Tutorials

Wiki HowTo

Visualization Utilities

Good Commits

Python and Albany

Clone this wiki locally