Skip to content
Glen Hansen edited this page Feb 11, 2019 · 2 revisions

These are the build instructions for Curie

Curie is a Cray XK4

  • Installed 1/2013
  • 52 compute nodes
  • 16-core 2.1GHz 64bit AMD Opteron 6200 CPU's (Interlagos)
  • 32 GB - 4 channels of DDR3 memory per compute node
  • NVIDIA K20x (Kepler)
  • 6GB integrated RDDR5 memory
  • 2688 732MHz cores
  • Gemini 1.2
  • SLES11

Set up your build environment

Here is the module load sequence used

	module purge
	module load sierra-devel/gcc-4.7.2-mpich2
	module load nodestat/2.2-1.0402.45287.1.2.gem
	module load craype-accel-nvidia35

Install the required Third Party Libraries (TPLs) for Trilinos

These install directions assume that a base install directory of /ascldap/users/gahanse is used

Building ZLib

  • Version 1.2.8

      cd /home/gahanse/src
      wget http://zlib.net/zlib-1.2.8.tar.gz
      tar -xvf zlib-1.2.8.tar.gz
      cd zlib-1.2.8
      CC=/opt/cray/xt-asyncpe/5.24/bin/cc CFLAGS="-O3" ./configure --prefix=/home/gahanse
      make install
    

Building HWLOC

  • Version 1.8

      ./configure --prefix=/home/gahanse CC=/opt/cray/xt-asyncpe/5.24/bin/cc CXX=/opt/cray/xt-asyncpe/5.24/bin/CC
    

Building HDF5

  • Version 1.8.14

      cd /ascldap/users/gahanse/src/
      wget http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.14.tar.bz2
      tar -xvf hdf5-1.8.14.tar.gz
      cd hdf5-1.8.14
      ./configure CC=mpicc FC=mpif90 CXX=mpicxx CXXFLAGS="-fPIC -O3" CFLAGS="-fPIC -O3" FCFLAGS="-fPIC -O3" \
      --enable-parallel --enable-shared=no --with-zlib=/ascldap/users/gahanse --prefix=/ascldap/users/gahanse
      make -j 8
      make install
    

Building Netcdf

  • Version of the day from the git repo. Note that hdf5-1.8.13 made some changes that are not reflected in the latest release, so we are using the votd for now.

      cd /ascldap/users/gahanse/src/
      git clone [email protected]:Unidata/netcdf-c.git
      cd netcdf-c
    
  • Edit the file include/netcdf.h. Make the following edits near line 228:

      #define NC_MAX_DIMS    65536    /* max dimensions per file */
      #define NC_MAX_ATTRS    8192    
      #define NC_MAX_VARS   524288    /* max variables per file */
      #define NC_MAX_NAME   256 
      #define NC_MAX_VAR_DIMS      8     /* max per variable dimensions */
    
  • Comment out the checks for HDF5 version on lines 501 - 510 in CMakeLists.txt. Netcdf incorrectly deduces that the hdf5 version is too old.

  • Comment out the MPI typedefs and defines at line 90 in include/ncdispatch.h, and add an include statement for mpi.h

      #include "mpi.h"
      /*typedef int MPI_Comm;
      typedef int MPI_Info;
      #define MPI_COMM_WORLD 0
      #define MPI_INFO_NULL 0 */
    
  • Now, configure using CMake and build:

      mkdir build
      cd build
      cmake \
      -DCMAKE_C_COMPILER=mpicc \
      -DCMAKE_C_FLAGS="-I/ascldap/users/gahanse/include -O3" \
      -DCMAKE_EXE_LINKER_FLAGS="-L/ascldap/users/gahanse/lib -O3" \
      -DCMAKE_INSTALL_PREFIX=/ascldap/users/gahanse \
      -DENABLE_DAP=OFF -DBUILD_SHARED_LIBS=OFF -DENABLE_FSYNC=OFF -DENABLE_CDMREMOTE=OFF \
      -DENABLE_DOXYGEN=OFF -DENABLE_NETCDF_4=ON -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_EXAMPLES=OFF \
      -DENABLE_TESTS=OFF -DCMAKE_SKIP_INSTALL_RPATH=ON \
      -DZLIB_INCLUDE_DIRS=/ascldap/users/gahanse/include \
      -DZLIB_LIBRARY=/ascldap/users/gahanse/lib/libz.a \
      -DBUILD_UTILITIES=OFF \
      ..
    
      make -j 8 
      make install
    

Building boost

  • Version 1.57.0

      cd /ascldap/users/gahanse/src/
      wget http://sourceforge.net/projects/boost/files/boost/1.57.0/boost_1_57_0.tar.gz/download
      tar -xvf boost_1_57_0.tar.gz
      cd boost_1_57_0
    
  • Build boost with the commands

      echo "using gcc : : : <compileflags>-I/opt/cray/mpt/6.1.1/gni/mpich2-gnu/47/include <linkflags>-L/opt/cray/mpt/6.1.1/gni/mpich2-gnu/47/lib ;" >> ./tools/build/user-config.jam
      echo "using mpi : : <find-shared-library>mpichcxx_gnu_47 ;" >> ./tools/build/user-config.jam
      ./bootstrap.sh \
      --with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \
      --prefix=/home/gahanse
      ./b2 -j 8
      ./b2 -j 8 install
    

Building ParMetis

  • Version 4.0.3

      cd /home/gahanse/src/
      wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz
      tar -xvf parmetis-4.0.3.tar.gz
      cd parmetis-4.0.3
    
  • Edit metis.h to use an IDXTYPEWIDTH of 64

      cd metis/include
      vi metis.h
      #define IDXTYPEWIDTH 64
    
  • Run CMake to build parmetis

      cd /home/gahanse/src/parmetis-4.0.3
      cd build
      export metis=/home/gahanse/src/parmetis-4.0.3/metis
    
      cmake \
      -DCMAKE_INSTALL_PREFIX=/home/gahanse \
      -DMETIS_PATH=$metis \
      -DGKLIB_PATH=$metis/GKlib \
      -DCMAKE_C_COMPILER=/opt/cray/xt-asyncpe/5.24/bin/cc \
      -DCMAKE_CXX_COMPILER=/opt/cray/xt-asyncpe/5.24/bin/CC \
      -DMPI_INCLUDE_PATH=/opt/cray/mpt/6.1.1/gni/mpich2-gnu/47/include \
      ..
    
      make -j 8
      make install
    
      cp libmetis/libmetis.a /home/gahanse/lib
      cp $metis/include/metis.h /home/gahanse/include
    

SuperLU

  • Version 4.3

      cd /home/gahanse/src
      wget http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_4.3.tar.gz
      tar -xvf superlu_4.3.tar.gz
      cd SuperLU_4.3
    
      mkdir /home/gahanse/SuperLU_4.3
      mkdir /home/gahanse/SuperLU_4.3/include
      mkdir /home/gahanse/SuperLU_4.3/lib
    
  • Edit make.inc for your machine and environment

      PLAT = _linux
      SuperLUroot = /home/gahanse/SuperLU_4.3
      SUPERLULIB      = $(SuperLUroot)/lib/libsuperlu_4.3.a
      TMGLIB          = libtmglib.a
      BLASDEF     = -DUSE_VENDOR_BLAS
      BLASLIB     = -L/opt/intel/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
      LIBS        = $(SUPERLULIB) $(BLASLIB)
      ARCH         = ar
      ARCHFLAGS    = cr
      RANLIB       = ranlib
      CC           = /opt/cray/xt-asyncpe/5.24/bin/cc
      CFLAGS       = -DPRNTlevel=0 -O3
      NOOPTS       =
      FORTRAN      = /opt/cray/xt-asyncpe/5.24/bin/ftn
      FFLAGS       = -O3
      LOADER       = $(CC)
      LOADOPTS     =
      CDEFS        = -DAdd_
      MATLAB       =
    
  • Build SuperLU

      make -j 4
      cd SRC
      cp *.h /home/ghansen/FASTMath/SuperLU_4.3/include
      ar d home/ghansen/FASTMath/SuperLU_4.3/lib/libsuperlu_4.3.a dlamch.o slamch.o
    
  • The last command deletes the dlamch and slamch objects, which already exist in the LAPACK library on Vesta

Building Lame (optional)

  • Version 4.24.1

  • These instructions are for the modified Lame tarball (contact gahanse for info)

      cd /ascldap/users/gahanse/src/lame-4.24.1
      mkdir build-intel
      ../do-cmake-lame-intel
      make -j 8
    

Build Trilinos

  • Download Trilinos

      cd /ascldap/users/gahanse/src
      git clone [email protected]:/git/Trilinos
    
  • Download SCOREC

      cd /ascldap/users/gahanse/src/Trilinos
      git clone [email protected]:SCOREC/core.git SCOREC
    
  • Configure the Trilinos build

      mkdir build
      cd build
      ./do-configure
    
  • Example do-configure script executed above

      #!/bin/sh
      #
      export TRILINOS_HOME=/ascldap/users/gahanse/src/Trilinos
      BUILD_DIR=`pwd`
      INSTALL_DIR=/ascldap/users/gahanse/trilinos/MPI_REL
      BOOST_DIR=/ascldap/users/gahanse
      LIB_DIR=$BOOST_DIR
      MPI_BASE_DIR=/opt/openmpi-1.4-intel
      NETCDF=/ascldap/users/gahanse
      HDFDIR=/ascldap/users/gahanse
      PARMETISDIR=$BOOST_DIR
      SUPERLUDIR=$BOOST_DIR/SuperLU_4.3
      MKL_PATH=/opt/intel-12.1
      LABLAS_LIBRARIES="-L$MKL_PATH/lib/intel64 -Wl,--start-group $MKL_PATH/mkl/lib/intel64/libmkl_intel_lp64.a $MKL_PATH/mkl/lib/intel64/libmkl_core.a $MKL_PATH/mkl/lib/intel64/libmkl_sequential.a -Wl,--end-group -lpthread"
      
      cmake \
          -Wno-dev \
          -D Trilinos_CONFIGURE_OPTIONS_FILE:FILEPATH=$TRILINOS_HOME/sampleScripts/AlbanySettings.cmake \
          -D Trilinos_EXTRA_REPOSITORIES:STRING="SCOREC" \
          -D Trilinos_ENABLE_SCOREC:BOOL=ON \
          -D SCOREC_DISABLE_STRONG_WARNINGS:BOOL=ON \
          -D CMAKE_BUILD_TYPE:STRING=NONE \
          -D CMAKE_CXX_FLAGS:STRING="-O3 -xAVX -DNDEBUG" \
          -D CMAKE_C_FLAGS:STRING="-O3 -xAVX -DNDEBUG" \
          -D CMAKE_Fortran_FLAGS:STRING="-O3 -xAVX -DNDEBUG" \
          -D Rythmos_ENABLE_DEBUG:BOOL=ON \
      \
          -D TPL_ENABLE_MPI:BOOL=ON \
          -D MPI_BASE_DIR:PATH=$MPI_BASE_DIR \
      \
      	-D TPL_ENABLE_Pthread:BOOL=ON \
      \
          -D TPL_ENABLE_Boost:BOOL=ON \
          -D TPL_ENABLE_BoostLib:BOOL=ON \
          -D TPL_ENABLE_BoostAlbLib:BOOL=ON \
      	-D Boost_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
      	-D Boost_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
      	-D BoostLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
      	-D BoostLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
      	-D BoostAlbLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
      	-D BoostAlbLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
      \
          -D TPL_ENABLE_Netcdf:STRING=ON \
      	-D Netcdf_INCLUDE_DIRS:PATH=$NETCDF/include \
      	-D Netcdf_LIBRARY_DIRS:PATH=$NETCDF/lib \
          -D TPL_ENABLE_HDF5:STRING=ON \
          -D HDF5_INCLUDE_DIRS:PATH="${HDFDIR}/include" \
          -D TPL_HDF5_LIBRARIES:PATH="${HDFDIR}/lib/libhdf5_hl.a;${HDFDIR}/lib/libhdf5.a;${HDFDIR}/lib/libz.a" \
           -D Trilinos_EXTRA_LINK_FLAGS="-L${HDFDIR}/lib -lhdf5_hl -lhdf5 -lz -lm" \
      \
          -D TPL_ENABLE_Zlib:STRING=ON \
          -D Zlib_INCLUDE_DIRS:PATH=$HDFDIR/include \
          -D TPL_Zlib_LIBRARIES:PATH="$HDFDIR/lib/libz.a" \
      \
          -D TPL_ENABLE_BLAS:STRING=ON \
          -D TPL_ENABLE_LAPACK:STRING=ON \
          -D TPL_BLAS_LIBRARIES:STRING="$LABLAS_LIBRARIES" \
          -D TPL_LAPACK_LIBRARIES:STRING="$LABLAS_LIBRARIES" \
      \
          -D TPL_ENABLE_ParMETIS:STRING=ON \
          -D ParMETIS_INCLUDE_DIRS:PATH="${PARMETISDIR}/include" \
          -D ParMETIS_LIBRARY_DIRS:PATH="${PARMETISDIR}/lib" \
      \
      	-D CMAKE_INSTALL_PREFIX:PATH=$INSTALL_DIR \
      \
          -D Trilinos_ENABLE_TriKota:BOOL=OFF \
          -D DAKOTA_ENABLE_TESTS:BOOL=OFF \
      \
      $EXTRA_ARGS \
      ${TRILINOS_HOME}
      
      exit 0
    
  • Build Trilinos

      make -j 8
      make install
    

Build Albany

  • Clone the repo

      cd /ascldap/users/gahanse/src
      git clone [email protected]:SNLComputation/Albany.git
      cd Albany
    
  • Configure the Albany build

      mkdir build
      cd build
      ./do-configure
    
  • CMake configure script ./do-configure:

      #!/bin/sh
      
      export ALB_64BIT_INT=ON
      export ALB_EPETRA=OFF
      export ALB_SEE=OFF
      export ALB_ENABLE_SCOREC=ON
      export ALB_ENABLE_LCM=ON
      export ALB_ENABLE_LAME=ON
      export ALB_ENABLE_HYDRIDE=OFF
      export ALB_ENABLE_QCAD=OFF
      export ALB_ENABLE_MOR=OFF
      export ALB_ENABLE_ASCR=OFF
      export ALB_ENABLE_AERAS=OFF
      export ALB_ENABLE_FELIX=OFF
      export ALB_ENABLE_AlbanyCI=OFF
      export ALB_DEBUG=OFF
      export ALB_ENABLE_CHECK_FPE=ON
      export ALB_ENABLE_SPECULATIVE=OFF
      export ALB_ENABLE_SG_MP=OFF
      export ALB_ENABLE_PERF_TESTS=ON
      export TRILINOS_INSTALL_DIR=/ascldap/users/gahanse/trilinos/MPI_REL
      export ALB_LAME_INC_DIR=/ascldap/users/gahanse/src/lame-4.24.1/lame/include
      export ALB_LAME_LIB_DIR=/ascldap/users/gahanse/src/lame-4.24.1/lame/build_intel
      export ALB_VERBOSE=OFF
      
      cmake \
      -D ALBANY_TRILINOS_DIR:FILEPATH="$TRILINOS_INSTALL_DIR" \
      -D ENABLE_LCM:BOOL=${ALB_ENABLE_LCM} \
      -D ENABLE_AERAS:BOOL=${ALB_ENABLE_AERAS} \
      -D ENABLE_QCAD:BOOL=${ALB_ENABLE_QCAD} \
      -D ENABLE_HYDRIDE:BOOL=${ALB_ENABLE_HYDRIDE} \
      -D ENABLE_LCM_SPECULATIVE:BOOL=${ALB_ENABLE_SPECULATIVE} \
      -D ENABLE_LAME:BOOL=${ALB_ENABLE_LAME} \
      -D CMAKE_VERBOSE_MAKEFILE:BOOL=${ALB_VERBOSE} \
      -D ENABLE_DEBUGGING:BOOL=${ALB_DEBUG} \
      -D ENABLE_CHECK_FPE:BOOL=${ALB_ENABLE_CHECK_FPE} \
      -D ENABLE_SCOREC:BOOL=${ALB_ENABLE_SCOREC} \
      -D ENABLE_FELIX:BOOL=${ALB_ENABLE_FELIX} \
      -D ENABLE_MOR:BOOL=${ALB_ENABLE_MOR} \
      -D ENABLE_ALBANY_CI:BOOL=${ALB_ENABLE_AlbanyCI} \
      -D ENABLE_ASCR:BOOL=${ALB_ENABLE_ASCR} \
      -D ENABLE_SG_MP:BOOL=${ALB_ENABLE_SG_MP} \
      -D ENABLE_PERFORMANCE_TESTS:BOOL=${ALB_ENABLE_PERF_TESTS} \
      -D LAME_INCLUDE_DIR:FILEPATH="$ALB_LAME_INC_DIR" \
      -D LAME_LIBRARY_DIR:FILEPATH="$ALB_LAME_LIB_DIR" \
      -D ALBANY_CTEST_TIMEOUT:INTEGER=70 \
      -D ENABLE_64BIT_INT:BOOL=${ALB_64BIT_INT} \
      -D ENABLE_ALBANY_EPETRA_EXE:BOOL=${ALB_EPETRA} \
      -D ENABLE_SEE:BOOL=${ALB_SEE} \
      -D ALBANY_MPI_OPTIONS:BOOL=ON \
      -D ALBANY_MPI_EXEC:STRING="mpiexec" \
      -D ALBANY_MPI_EXEC_NUMPROCS_FLAG:STRING="-n" \
      -D ALBANY_MPI_EXEC_MAX_NUMPROCS:STRING="4" \
      -D ALBANY_MPI_LEADING_OPTIONS:STRING="" \
       \
      ..
    
  • Build Albany

      make -j 8
      make install
    
Clone this wiki locally