Support OpenMP offload with the NVIDIA compiler

These changes enable the ElectromagneticPIC OpenMP offload application to succeed (Particles/ElectromagneticPIC/Exec/OpenMP). It requires NVIDIA HPC SDK 21.3 or higher. Earlier versions of NVIDIA HPC SDK fail for one of two reasons: no interoperability between CUDA and OpenMP offload, or failure to support Fortran automatic arrays in OpenMP offload regions. I tested on Cori-GPU with nvhpc/21.3 and cuda/11.1.1.

Author: cdaley-lbl

Tools/GNUMake/Make.defs

@@ -849,7 +849,15 @@ f77EXETempDir	= $(TmpBuildDir)/f/$(optionsSuffix).EXE
 tmpEXETempDir	= $(TmpBuildDir)/t/$(optionsSuffix).EXE
 
 includes	= -I$(srcTempDir) -I. $(addprefix -I, $(INCLUDE_LOCATIONS)) $(addprefix -isystem , $(SYSTEM_INCLUDE_LOCATIONS))
-fincludes	= $(includes)
+ifeq ($(lowercase_comp),pgi)
+  # pgfortran-Error-Unknown switch: -isystem
+  fincludes = $(subst -isystem,-I,$(includes))
+else ifeq ($(lowercase_comp),nvhpc)
+  # nvfortran-Error-Unknown switch: -isystem
+  fincludes = $(subst -isystem,-I,$(includes))
+else
+  fincludes = $(includes)
+endif
 fmoddir         = $(objEXETempDir)
 
 amrexlib = $(objEXETempDir)/libamrex.a

Tools/GNUMake/comps/nvhpc.mak

@@ -19,6 +19,10 @@ gcc_minor_version = $(shell g++ -dumpfullversion -dumpversion | head -1 | sed -e
 
 COMP_VERSION = $(nvhpc_version)
 
+# -MP not supported by nvhpc and -MMD's output is put in the wrong directory
+USE_LEGACY_DEPFLAGS = TRUE
+DEPFLAGS =
+
 ########################################################################
 
 GENERIC_NVHPC_FLAGS =
@@ -27,6 +31,18 @@ ifeq ($(USE_OMP),TRUE)
   GENERIC_NVHPC_FLAGS += -mp -Minfo=mp
 endif
 
+ifeq ($(USE_OMP_OFFLOAD),TRUE)
+  # CUDA + NVIDIA OpenMP-offload requires CUDA 11.0 or later:
+  # "nvfortran-Fatal-Use of -Mcuda and -mp=gpu requires CUDA 11.0 or later"
+  # My Cori GPU interactive tests worked without needing to specify
+  # a CUDA version: I used nvhpc/21.3 and cuda/11.1.1 modules.
+  # (The USE_ACC code path below should be revisited)
+  GENERIC_NVHPC_FLAGS += -mp=gpu -Minfo=mp
+  ifneq ($(CUDA_ARCH),)
+    GENERIC_NVHPC_FLAGS += -gpu=cc$(CUDA_ARCH)
+  endif
+endif
+
 ifeq ($(USE_ACC),TRUE)
   GENERIC_NVHPC_FLAGS += -acc=gpu -Minfo=accel -mcmodel=medium
   ifneq ($(CUDA_ARCH),)
@@ -208,7 +224,9 @@ F90FLAGS += $(GENERIC_NVHPC_FLAGS)
 
 ########################################################################
 
-override XTRALIBS += -lstdc++ -latomic -lnvf
+# Add -lrt for the missing "aio_return" symbol
+# /usr/common/software/sles15_cgpu/nvhpc/21.3/Linux_x86_64/21.3/compilers/lib/libnvf.a:async.o:                 U aio_return
+override XTRALIBS += -lstdc++ -latomic -lnvf -lrt
 
 LINK_WITH_FORTRAN_COMPILER ?= $(USE_F_INTERFACES)
 

Tools/libamrex/mkconfig.py

@@ -46,6 +46,9 @@ def doit(defines, undefines, comp, allow_diff_comp):
         elif comp == "pgi":
             comp_macro = "__PGI"
             comp_id    = "PGI"
+        elif comp == "nvhpc":
+            comp_macro = "__NVCOMPILER"
+            comp_id    = "NVHPC"
         elif comp == "llvm":
             comp_macro = "__llvm__"
             comp_id    = "Clang/LLVM"
@@ -89,7 +92,7 @@ def doit(defines, undefines, comp, allow_diff_comp):
                         default="")
     parser.add_argument("--comp",
                         help="compiler",
-                        choices=["gnu","intel","cray","pgi","llvm","nag","nec","ibm","hip","dpcpp"])
+                        choices=["gnu","intel","cray","pgi","nvhpc","llvm","nag","nec","ibm","hip","dpcpp"])
     parser.add_argument("--allow-different-compiler",
                         help="allow an application to use a different compiler than the one used to build libamrex",
                         choices=["TRUE","FALSE"])