Added Makefile.amdclang. Added best option to make.hip and make.andclang to access fastest compile option

Author: Ram3sh

src/Makefile.amdclang

@@ -0,0 +1,46 @@
+include configs/make.amdclang
+builddir=./
+ifdef EXTRA_CXX_FLAGS
+   CXXFLAGS2 += $(EXTRA_CXX_FLAGS2)
+   CXXFLAGS += $(EXTRA_CXX_FLAGS)
+endif
+
+ifdef EXTRA_LINK_FLAGS
+   linklibs += $(EXTRA_LINK_FLAGS)
+endif
+
+OBJSW4 = 
+
+OBJ  = curvitest.o curvilinear4sgc.o Events.o
+
+
+# prefix object files with build directory
+FSW4 = $(addprefix $(builddir)/,$(OBJSW4))
+FOBJ = $(addprefix $(builddir)/,$(OBJ)) $(addprefix $(builddir)/,$(QUADPACK))
+
+# prefix
+sw4ck: $(FSW4) $(FOBJ)
+	@echo "*** Configuration file: '" $(foundincfile) "' ***"
+	@echo "********* User configuration variables **************"
+	@echo "debug=" $(debug) " proj=" $(proj) " etree=" $(etree) " SW4ROOT"= $(SW4ROOT) 
+	@echo "CXX=" $(CXX) "EXTRA_CXX_FLAGS"= $(EXTRA_CXX_FLAGS)
+	@echo "FC=" $(FC) " EXTRA_FORT_FLAGS=" $(EXTRA_FORT_FLAGS)
+	@echo "EXTRA_LINK_FLAGS"= $(EXTRA_LINK_FLAGS)
+	@echo "******************************************************"
+	cd $(builddir); $(LINKER) $(LINKFLAGS) -o $@ $(OBJ) $(linklibs)
+# test: linking with openmp for the routine rhs4sgcurv.o
+#	cd $(builddir); $(CXX) $(CXXFLAGS) -qopenmp -o $@ main.o $(OBJ) $(QUADPACK) $(linklibs)
+	@echo "*** Build directory: " $(builddir) " ***"
+ 
+$(builddir)/%.o:./%.C
+	 cd $(builddir); $(CXX) $(CXXFLAGS) -c $< 
+
+clean:
+	rm sw4ck *.o *.bc *.s *.cui
+
+format:
+	clang-format -style Google -i src/*.C
+	clang-format -style Google -i src/*.h
+
+tags:
+	etags -o src/TAGS src/*.C src/*.h 

src/configs/make.amdclang

@@ -0,0 +1,41 @@
+#use make -f Makefile.hipcc
+
+LINKER = amdclang++
+CXX = $(PREP) amdclang++
+RAJA_LOCATION = /home/users/coe0145/RAJA/Dev2/raja/install_90a
+
+HIP_ROOT_DIR = $(HIP_PATH)
+
+HSA_ROOT_DIR = ${ROCM_PATH}/hsa
+
+GCC_LINK_LINE = -L $(HIP_ROOT_DIR)/lib -L $(HSA_ROOT_DIR)/lib -lhsa-runtime64  -lm -lstdc++ -ldl
+
+
+MORE_FLAGS = -I${ROCM_PATH}/include -isystem ${ROCM_PATH}/llvm/lib/clang/13.0.0/include/.. -isystem ${ROCM_PATH}/hsa/include -isystem ${ROCM_PATH}/hip/include -D__HIP_ARCH_GFX90a__=1 -O3 -x hip -Winconsistent-missing-override -DNO_RAJA=1 -Wall -Wno-unknown-pragmas --offload-arch=gfx90a -fvectorize -fslp-vectorize -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false
+
+
+LINKFLAGS =  -std=c++11 -fgpu-rdc --offload-arch=gfx90a --hip-link  -L ${ROCM_PATH}/lib -lamdhip64
+EXTRA_CXX_FLAGS =  -O3  -x hip  -std=c++11 -I$(RAJA_LOCATION)/include  -DENABLE_HIP $(MORE_FLAGS) -Winconsistent-missing-override 
+
+EXTRA_CXX_FLAGS2 =  -O3  -std=c++11 -I $(CUDA_HOME)/include -I$(RAJA_LOCATION)/include  -DRAJA_USE_RESTRICT_PTR -DCUDA_CODE -DENABLE_HIP   $(MORE_FLAGS)
+
+EXTRA_LINK_FLAGS = -L $(RAJA_LOCATION)/lib $(GCC_LINK_LINE) 
+
+ifeq ($(best),yes)
+amd_unroll_fix=yes
+magic_sync=yes
+endif
+
+ifeq ($(amd_unroll_fix),yes)
+MORE_FLAGS+=  -DAMD_UNROLL_FIX=1
+endif
+
+ifeq ($(magic_sync),yes)
+MORE_FLAGS+=  -DMAGIC_SYNC=1
+endif
+
+ifeq ($(register_count),yes)
+MORE_FLAGS+=  --save-temps
+else
+MORE_FLAGS+= -fgpu-rdc
+endif

src/configs/make.hip

@@ -11,7 +11,7 @@ HSA_ROOT_DIR = ${ROCM_PATH}/hsa
 GCC_LINK_LINE = -L $(HIP_ROOT_DIR)/lib -L $(HSA_ROOT_DIR)/lib -lhsa-runtime64  -lm -lstdc++ -ldl
 
 
-MORE_FLAGS = -I${ROCM_PATH}/include -isystem ${ROCM_PATH}/llvm/lib/clang/12.0.0/include/.. -isystem ${ROCM_PATH}/hsa/include -isystem ${ROCM_PATH}/hip/include -D__HIP_ARCH_GFX90a__=1 --cuda-gpu-arch=gfx90a -O3 -x hip -Winconsistent-missing-override --amdgpu-target=gfx90a -DNO_RAJA=1 -Wall -Wno-unknown-pragmas
+MORE_FLAGS = -I${ROCM_PATH}/include -isystem ${ROCM_PATH}/llvm/lib/clang/12.0.0/include/.. -isystem ${ROCM_PATH}/hsa/include -isystem ${ROCM_PATH}/hip/include -D__HIP_ARCH_GFX90a__=1 --cuda-gpu-arch=gfx90a -O3 -x hip -Winconsistent-missing-override --amdgpu-target=gfx90a -DNO_RAJA=1 -Wall -Wno-unknown-pragmas 
 
 
 LINKFLAGS =  -std=c++11  --amdgpu-target=gfx90a -fgpu-rdc
@@ -21,6 +21,11 @@ EXTRA_CXX_FLAGS2 =  -O3  -std=c++11 -I $(CUDA_HOME)/include -I$(RAJA_LOCATION)/i
 
 EXTRA_LINK_FLAGS = -L $(RAJA_LOCATION)/lib $(GCC_LINK_LINE) 
 
+ifeq ($(best),yes)
+amd_unroll_fix=yes
+magic_sync=yes
+endif
+
 ifeq ($(amd_unroll_fix),yes)
 MORE_FLAGS+=  -DAMD_UNROLL_FIX=1
 endif
@@ -34,3 +39,4 @@ MORE_FLAGS+=  --save-temps
 else
 MORE_FLAGS+= -fgpu-rdc
 endif
+

src/curvitest.C

@@ -314,6 +314,12 @@ void promo_version(){
   std::stringstream s;
 #ifdef ENABLE_HIP
   s<<"HIP("<<HIP_VERSION_MAJOR<<"."<<HIP_VERSION_MINOR<<"."<<HIP_VERSION_PATCH<<")\n";
+#ifdef AMD_UNROLL_FIX
+  s<<"AMD unroll fix enabled\n";
+#endif
+#ifdef MAGIC_SYNC
+  s<<"Magic sync enabled\n";
+#endif
 #elif ENABLE_CUDA
   s<<"CUDA("<<CUDA_VERSION<<")\n";
 #else
@@ -322,6 +328,7 @@ void promo_version(){
 #ifndef NO_RAJA
 s<<"RAJA("<<RAJA_VERSION_MAJOR<<"."<<RAJA_VERSION_MINOR<<"."<<RAJA_VERSION_PATCHLEVEL<<")\n";
 #endif
+s<<"\n";
 std::cout<<s.str();
 
 }

src/run_flux

@@ -0,0 +1 @@
+flux mini run -n 1 -c 8 -g 1 -o gpu-affinity=per-task -o cpu-affinity=per-task ./sw4ck sw4ck.in