diff --git a/src/dataService.py b/src/dataService.py index 3fe03cf26bed92fce2638ff9f2d6fa877dcbaa89..c110a4f6b02e087ed9a90bc3e1ee86858f8d4e83 100644 --- a/src/dataService.py +++ b/src/dataService.py @@ -150,7 +150,7 @@ class DataService(object,metaclass=Singleton): def get_clean_cmd(self): return f''' -set -e +{self.get_env()} cd {DataService.build_dir} {DataService.clean_cmd} ''' @@ -182,4 +182,4 @@ cd {DataService.build_dir} {self.get_env()} cd {DataService.case_dir} {self.get_run()} -''' \ No newline at end of file +''' diff --git a/templates/CP2K/8.2/data.CP2K.arm.gpu.config b/templates/CP2K/8.2/data.CP2K.arm.gpu.config index 63c1d24bf2caa2459f981724224589f97688eada..c6dbfa66c1bb6031f0e98c20c2adad04a18e7821 100644 --- a/templates/CP2K/8.2/data.CP2K.arm.gpu.config +++ b/templates/CP2K/8.2/data.CP2K.arm.gpu.config @@ -5,6 +5,8 @@ cp2k/8.2 https://github.com/cp2k/cp2k/releases/download/v8.2.0/cp2k-8.2.tar.bz2 [DEPENDENCY] +set -e +set -x ./jarvis -install kgcc/9.3.1 com module purge module use ./software/modulefiles @@ -15,16 +17,19 @@ export FC=`which gfortran` ./jarvis -install openmpi/4.1.2 gcc module load openmpi4/4.1.2 ./jarvis -install gmp/6.2.0 gcc +module load gmp/6.2.0 ./jarvis -install boost/1.72.0 gcc +module load boost/1.72.0 ./jarvis -install libint/2.6.0 gcc+mpi ./jarvis -install fftw/3.3.8 gcc+mpi +./jarvis -install kml/1.4.0/gcc gcc ./jarvis -install openblas/0.3.18 gcc -module load openblas/0.3.18 -./jarvis -install scalapack/2.1.0 gcc+mpi +./jarvis -install scalapack/2.1.0/kml gcc+mpi ./jarvis -install spglib/1.16.0 gcc ./jarvis -install libxc/5.1.4 gcc ./jarvis -install gsl/2.6 gcc module load gsl/2.6 +module load openblas/0.3.18 ./jarvis -install plumed/2.6.2 gcc+mpi ./jarvis -install libvori/21.04.12 gcc #release CP2K @@ -32,24 +37,94 @@ tar -jxvf downloads/cp2k-8.2.tar.bz2 [ENV] module purge +module use ./software/modulefiles module load kgcc9/9.3.1 module load openmpi4/4.1.2 +module load scalapack-kml/2.1.0 module load gsl/2.6 +export CUDA_INCLUDE_DIRS=/usr/local/cuda/include +export CUDA_CUDART_LIBRARY=/usr/local/cuda/lib64/libcudart.so +export CUDA_HOME=/usr/local/cuda +export PATH=$CUDA_HOME/bin:$PATH +export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH +export C_INCLUDE_PATH=$CUDA_HOME/include:$C_INCLUDE_PATH +export LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH [APP] app_name = CP2K build_dir = ${JARVIS_ROOT}/cp2k-8.2/ -binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/local-cuda/ +binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/arm-cuda/ case_dir = ${JARVIS_ROOT}/cp2k-8.2/benchmarks/QS/ [BUILD] -make -j 128 ARCH=local-cuda VERSION=psmp +cat >arch/arm-cuda.psmp <<"EOF" +NVCC = /usr/local/cuda/bin/nvcc +NVFLAGS = -O3 -g -w --std=c++11 -arch=sm_80 +OFFLOAD_TARGET = cuda +GPUVER = V100 + +CC = mpicc +CXX = mpic++ +FC = mpif90 +LD = mpif90 +AR = ar -r + +MPI_PATH = ${JARVIS_ROOT}/software/libs/kgcc9/openmpi4 +GNU_PATH = ${JARVIS_ROOT}/software/libs/kgcc9 + +include $(MPI_PATH)/plumed/2.6.2/lib/plumed/src/lib/Plumed.inc.static + +FFTW_INC = $(MPI_PATH)/fftw/3.3.8/include +FFTW_LIB = $(MPI_PATH)/fftw/3.3.8/lib + +LIBINT_INC = $(MPI_PATH)/libint/2.6.0/include +LIBINT_LIB = $(MPI_PATH)/libint/2.6.0/lib + +LIBVORI_LIB = $(GNU_PATH)/libvori/21.04.12/lib + +LIBXC_INC = $(GNU_PATH)/libxc/5.1.4/include +LIBXC_LIB = $(GNU_PATH)/libxc/5.1.4/lib + +SPGLIB_INC = $(GNU_PATH)/spglib/1.16.0/include +SPGLIB_LIB = $(GNU_PATH)/spglib/1.16.0/lib64 + +CFLAGS = -O3 -g -mtune=native -fopenmp + +DFLAGS = -D__FFTW3 -D__LIBINT -D__LIBXC -D__GRID_CUDA +DFLAGS += -D__MPI_VERSION=3 -D__PLUMED2 -D__SPGLIB +DFLAGS += -D__parallel -D__SCALAPACK +DFLAGS += -D__CUDA -D__DBCSR_ACC +FCFLAGS = $(CFLAGS) $(DFLAGS) +FCFLAGS += -ffree-form -ffree-line-length-none +FCFLAGS += -fopenmp +FCFLAGS += -ftree-vectorize -funroll-loops -std=f2008 +FCFLAGS += -I$(FFTW_INC) -I$(LIBINT_INC) -I$(LIBXC_INC) + +LDFLAGS = $(FCFLAGS) -static-libgfortran + +LIBS = $(PLUMED_DEPENDENCIES) -L$(GNU_PATH)/gsl/2.6/lib/ -lgsl -lgslcblas -lz +LIBS += $(LIBVORI_LIB)/libvori.a +LIBS += $(LIBXC_LIB)/libxcf03.a +LIBS += $(LIBXC_LIB)/libxc.a +LIBS += $(LIBINT_LIB)/libint2.a +LIBS += $(SPGLIB_LIB)/libsymspg.a +LIBS += $(FFTW_LIB)/libfftw3.a +LIBS += $(FFTW_LIB)/libfftw3_threads.a +LIBS += -L$(MPI_PATH)/scalapack-kml/2.1.0/lib -lscalapack +LIBS += -L/usr/local/kml/lib/kblas/omp -lkblas +LIBS += -L/usr/local/kml/lib -lklapack_full +LIBS += -ldl -lpthread -lstdc++ +LIBS += -L/usr/local/cuda/lib64 -lnvToolsExt -lnvrtc -lcudart -lcublas -lcufft -lnvrtc -lcuda +EOF + +make -j 128 ARCH=arm-cuda VERSION=psmp +ldd exe/arm-cuda/cp2k.psmp [CLEAN] -make -j 128 ARCH=local-cuda VERSION=psmp clean +make -j 128 ARCH=arm-cuda VERSION=psmp realclean [RUN] -run = numactl -C 0-63 mpirun --allow-run-as-root -x CUDA_VISIBLE_DEVICES=0,1 -np 64 -x OMP_NUM_THREADS=1 +run = mpirun --allow-run-as-root --mca btl ^openib -np 64 -x OMP_NUM_THREADS=1 binary = cp2k.psmp H2O-256.inp nodes = 1