Dev/2decomp gitsub

.gitignore

@@ -1,4 +1,6 @@
 *.o
 *.mod
 xcompact3d
-*.dat
+*.dat
+2decomp
+TAGS

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "decomp2d"]
+	path = decomp2d
+	url = [email protected]:xcompact3d/2decomp_fft.git

Makefile

@@ -7,8 +7,9 @@
 #   -DDEBG        - debuggin xcompact3d.f90
 # generate a Git version string
 GIT_VERSION := $(shell git describe --tag --long --always)
+BUILD =
 
-DEFS = -DDOUBLE_PREC -DVERSION=\"$(GIT_VERSION)\"
+DEFS ?= -DDOUBLE_PREC
 
 LCL = local# local,lad,sdu,archer
 IVER = 17# 15,16,17,18
@@ -17,74 +18,59 @@ FFT = generic# generic,fftw3,mkl
 
 #######CMP settings###########
 ifeq ($(CMP),intel)
-FC = mpiifort
-#FFLAGS = -fpp -O3 -xHost -heap-arrays -shared-intel -mcmodel=large -safe-cray-ptr -g -traceback
-FFLAGS = -fpp -O3 -xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr -I$(MPI_ROOT)/lib
-##debuggin test: -check all -check bounds -chintel eck uninit -gen-interfaces -warn interfaces
+	FC = mpiifort
+	#FFLAGS = -fpp -O3 -xHost -heap-arrays -shared-intel -mcmodel=large -safe-cray-ptr -g -traceback
+	FFLAGS = -fpp -O3 -xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr -I$(MPI_ROOT)/lib
+	##debuggin test: -check all -check bounds -chintel eck uninit -gen-interfaces -warn interfaces
 else ifeq ($(CMP),gcc)
-FC = mpif90
-#FFLAGS = -O3 -funroll-loops -floop-optimize -g -Warray-bounds -fcray-pointer -x f95-cpp-input
-FFLAGS = -cpp -O3 -funroll-loops -floop-optimize -g -Warray-bounds -fcray-pointer -fbacktrace -ffree-line-length-none
-#-ffpe-trap=invalid,zero
+	FC = mpif90
+	FFLAGS = -cpp
+	ifeq ($(BUILD), debug)
+		FFLAGS += -g3 -Og
+		FFLAGS += -ffpe-trap=invalid,zero
+	else
+		FFLAGS += -g -O3
+		FFLAGS += -funroll-loops -floop-optimize
+	endif
+	FFLAGS += -Warray-bounds -fcray-pointer -fbacktrace -ffree-line-length-none
 else ifeq ($(CMP),nagfor)
-FC = mpinagfor
-FFLAGS = -fpp
+	FC = mpinagfor
+	FFLAGS = -fpp
 else ifeq ($(CMP),cray)
-FC = ftn
-FFLAGS = -eF -g -O3 -N 1023
+	FC = ftn
+	FFLAGS = -eF -g -O3 -N 1023
 endif
 
-
 MODDIR = ./mod
 DECOMPDIR = ./decomp2d
 SRCDIR = ./src
 
 ### List of files for the main code
-SRCDECOMP = $(DECOMPDIR)/decomp_2d.f90 $(DECOMPDIR)/glassman.f90 $(DECOMPDIR)/fft_$(FFT).f90 $(DECOMPDIR)/io.f90
-OBJDECOMP = $(SRCDECOMP:%.f90=%.o)
-SRC = $(SRCDIR)/module_param.f90 $(SRCDIR)/variables.f90 $(SRCDIR)/poisson.f90 $(SRCDIR)/derive.f90 $(SRCDIR)/schemes.f90 $(SRCDIR)/parameters.f90 #$(SRCDIR)/*.f90
+SRC = $(SRCDIR)/module_param.f90 $(SRCDIR)/variables.f90 $(SRCDIR)/poisson.f90 $(SRCDIR)/derive.f90 $(SRCDIR)/schemes.f90 $(SRCDIR)/parameters.f90
 OBJ = $(SRC:%.f90=%.o)
 SRC = $(SRCDIR)/module_param.f90 $(SRCDIR)/variables.f90 $(SRCDIR)/poisson.f90 $(SRCDIR)/derive.f90 $(SRCDIR)/schemes.f90 $(SRCDIR)/navier.f90 $(SRCDIR)/parameters.f90 $(SRCDIR)/mom.f90 $(SRCDIR)/case.f90 $(SRCDIR)/transeq.f90 $(SRCDIR)/xcompact3d.f90
 
-
-#######FFT settings##########
-ifeq ($(FFT),fftw3)
-  #FFTW3_PATH=/usr
-  #FFTW3_PATH=/usr/lib64
-  FFTW3_PATH=/usr/local/Cellar/fftw/3.3.7_1
-  INC=-I$(FFTW3_PATH)/include
-  LIBFFT=-L$(FFTW3_PATH) -lfftw3 -lfftw3f
-else ifeq ($(FFT),fftw3_f03)
-  FFTW3_PATH=/usr                                #ubuntu # apt install libfftw3-dev
-  #FFTW3_PATH=/usr/lib64                         #fedora # dnf install fftw fftw-devel
-  #FFTW3_PATH=/usr/local/Cellar/fftw/3.3.7_1     #macOS  # brew install fftw
-  INC=-I$(FFTW3_PATH)/include
-  LIBFFT=-L$(FFTW3_PATH)/lib -lfftw3 -lfftw3f
-else ifeq ($(FFT),generic)
-  INC=
-  LIBFFT=
-else ifeq ($(FFT),mkl)
-  SRCDECOMP := $(DECOMPDIR)/mkl_dfti.f90 $(SRCDECOMP)
-  LIBFFT=-Wl,--start-group $(MKLROOT)/lib/intel64/libmkl_intel_lp64.a $(MKLROOT)/lib/intel64/libmkl_sequential.a $(MKLROOT)/lib/intel64/libmkl_core.a -Wl,--end-group -lpthread
-	INC=-I$(MKLROOT)/include
-endif
-
 #######OPTIONS settings###########
-OPT = -I$(SRCDIR) -I$(DECOMPDIR) $(FFLAGS)
+OPT = -I$(SRCDIR) -I$(DECOMP_INCDIR) $(FFLAGS)
 LINKOPT = $(FFLAGS)
 #-----------------------------------------------------------------------
 # Normally no need to change anything below
 
-all: xcompact3d
+DECOMP_LIB = 2decomp_fft
+DECOMP_LIBDIR = $(DECOMPDIR)/lib
+DECOMP_INCDIR = $(DECOMPDIR)/include
+DECOMP.A = $(DECOMP_LIBDIR)/lib$(DECOMP_LIB).a
+include $(DECOMPDIR)/src/Makefile.inc
 
-xcompact3d : $(OBJDECOMP) $(OBJ)
-	$(FC) -o $@ $(LINKOPT) $(OBJDECOMP) $(OBJ) $(LIBFFT)
+DEFS += -DVERSION=\"$(GIT_VERSION)\"
 
-$(OBJDECOMP):$(DECOMPDIR)%.o : $(DECOMPDIR)%.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(DEFS2) $(INC) -c $<
-	mv $(@F) ${DECOMPDIR}
-	#mv *.mod ${DECOMPDIR}
+all: xcompact3d
 
+xcompact3d : $(DECOMP.A) $(OBJ)
+	$(FC) -o $@ $(LINKOPT) $(OBJ) -L$(DECOMP_LIBDIR) -l$(DECOMP_LIB) $(LIBFFT)
+
+$(DECOMP.A):
+	make -C $(DECOMPDIR) F90=$(FC) OPTIONS="$(FFLAGS) $(DEFS) $(DEFS2)" lib
 
 $(OBJ):$(SRCDIR)%.o : $(SRCDIR)%.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(DEFS2) $(INC) -c $<
@@ -104,10 +90,13 @@ post:
 
 
 clean:
-	rm -f $(DECOMPDIR)/*.o $(DECOMPDIR)/*.mod
 	rm -f $(SRCDIR)/*.o $(SRCDIR)/*.mod
 	rm -f *.o *.mod xcompact3d post
 
+.PHONY: clean-decomp
+clean-decomp:
+	make -C $(DECOMPDIR) clean
+
 .PHONY: cleanall
-cleanall: clean
+cleanall: clean clean-decomp
 	rm -f *~ \#*\# out/* data/* stats/* planes/* *.xdmf *.log *.out nodefile core sauve*

README.md

@@ -34,3 +34,32 @@ For benchmarking `$test` mode should be disabled, its intention is to validate c
 implementations of the compact finite difference scheme solvers.
 
 Note this code is a very stripped down version of Xcompact3d, it is intended for profiling only.
+
+## 2decomp&fft
+
+Like Xcompact3d, x3div builds upon the 2decomp&fft library - rather than copy the code, we now
+have a git submodule tracking our fork [2decomp&fft](https://github.com/xcompact3d/2decomp_fft)
+which tracks the upstream repo [2decomp&fft-upstream](https://github.com/numericalalgorithmsgroup/2decomp_fft).
+This means we can share code more easily (and benefit from others' contributions).
+
+For detailed instructions on using git submodules see [git docs](https://git-scm.com/book/en/v2/Git-Tools-Submodules)
+however if you are not planning to work on 2decomp&fft itself the following should suffice:
+
+1) Initial clone of x3div ``git clone https://github.com/xcompact3d/x3div``
+2) Initialise and update the 2decomp&fft submodule ``cd x3div && git submodule init && git submodule update``
+
+After which you can continue to build as normal (running ``make`` will first call ``make`` on the 2decomp&fft submodule
+and then link the resulting library into x3div).
+To ensure you receive the latest changes to 2decomp&fft run ``git submodule update --remote`` periodically, note that you
+can also work within the ``decomp2d/`` directory as though it were a standalone git project.
+
+Note that variables are passed down by ``make``, therefore if you have installed ``ffte`` at
+``${FFTE_DIR}`` then you can build against this by running
+
+``
+make FFT=ffte FFTE_PATH=${FFTE_DIR}
+``
+
+where ``FFTE_PATH`` is used by 2decomp&fft to link the appropriate library - see
+``decomp2d/src/Makefile.inc`` for different FFT library options.
+By default ``FFT=generic`` and no external libraries are required.