From cfc8e7abce89b4f5575c0c46c7a8d2e8b35e81c4 Mon Sep 17 00:00:00 2001
From: Qiyu8 <fangchunlin@huawei.com>
Date: Tue, 2 Aug 2022 19:59:25 +0800
Subject: [PATCH 1/3] add cp2k gpu config

---
 templates/CP2K/8.2/Linux-arm-cuda.psmp      | 52 +++++++++++++++++++++
 templates/CP2K/8.2/data.CP2K.arm.gpu.config | 24 +++++++---
 2 files changed, 70 insertions(+), 6 deletions(-)
 create mode 100644 templates/CP2K/8.2/Linux-arm-cuda.psmp

diff --git a/templates/CP2K/8.2/Linux-arm-cuda.psmp b/templates/CP2K/8.2/Linux-arm-cuda.psmp
new file mode 100644
index 0000000..b85eaea
--- /dev/null
+++ b/templates/CP2K/8.2/Linux-arm-cuda.psmp
@@ -0,0 +1,52 @@
+# Tested with: GFortran 7.4.0, MPICH 3.3, LAPACK 3.8.0, ScaLAPACK 2.0.2, FFTW 3.3, Libint 2.6.0, Libxc 4.3.4, libxsmm 1.14, ELPA 2019.05.001, PLUMED 2.5.2, SPGLIB 1.12.2
+# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2019)
+NVCC    = /usr/local/cuda/bin/nvcc
+DFLAGS += -D__ACC -D__DBCSR_ACC -D__PW_CUDA
+LIBS   += -L/usr/local/cuda/lib64 -lnvToolsExt -lnvrtc -lcudart -lcublas -lcufft -lnvrtc
+CC          = mpicc
+FC          = mpif90
+LD          = mpif90
+AR          = ar -r
+
+MPI_PATH        = path1
+GNU_PATH        = path2
+
+include       $(MPI_PATH)/plumed/2.6.2/lib/plumed/src/lib/Plumed.inc.static
+
+FFTW_INC    = $(MPI_PATH)/fftw/3.3.8/include
+FFTW_LIB    = $(MPI_PATH)/fftw/3.3.8/lib
+
+LIBINT_INC  = $(MPI_PATH)/libint/2.6.0/include
+LIBINT_LIB  = $(MPI_PATH)/libint/2.6.0/lib
+
+LIBXC_INC   = $(GNU_PATH)/libxc/5.1.4/include
+LIBXC_LIB   = $(GNU_PATH)/libxc/5.1.4/lib
+
+SPGLIB_INC  = $(GNU_PATH)/spglib/1.16.0/include
+SPGLIB_LIB  = $(GNU_PATH)/spglib/1.16.0/lib64
+
+CFLAGS      = -O3 -g -mtune=native -fopenmp
+
+DFLAGS      = -D__FFTW3 -D__LIBINT -D__LIBXC
+DFLAGS     += -D__MPI_VERSION=3 -D__PLUMED2 -D__SPGLIB
+DFLAGS     += -D__parallel -D__SCALAPACK
+
+FCFLAGS     = $(CFLAGS) $(DFLAGS)
+FCFLAGS    += -ffree-form -ffree-line-length-none
+FCFLAGS    += -fopenmp
+FCFLAGS    += -ftree-vectorize -funroll-loops -std=f2008
+FCFLAGS    += -I$(FFTW_INC) -I$(LIBINT_INC) -I$(LIBXC_INC)
+
+LDFLAGS     = $(FCFLAGS) -static-libgfortran
+
+LIBS        = $(PLUMED_DEPENDENCIES) -L$(GNU_PATH)/gsl/2.6/lib/ -lgsl -lgslcblas -lz
+LIBS       += $(LIBXC_LIB)/libxcf03.a
+LIBS       += $(LIBXC_LIB)/libxc.a
+LIBS       += $(LIBINT_LIB)/libint2.a
+LIBS       += $(SPGLIB_LIB)/libsymspg.a
+LIBS       += $(FFTW_LIB)/libfftw3.a
+LIBS       += $(FFTW_LIB)/libfftw3_threads.a
+LIBS       += -L$(MPI_PATH)/scalapack-kml/2.1.0/lib -lscalapack
+LIBS       += -L/usr/local/kml/lib/kblas/omp -lkblas
+LIBS       += -L/usr/local/kml/lib -lklapack_full
+LIBS       += -ldl -lpthread -lstdc++
diff --git a/templates/CP2K/8.2/data.CP2K.arm.gpu.config b/templates/CP2K/8.2/data.CP2K.arm.gpu.config
index 63c1d24..5a4f8c8 100644
--- a/templates/CP2K/8.2/data.CP2K.arm.gpu.config
+++ b/templates/CP2K/8.2/data.CP2K.arm.gpu.config
@@ -5,6 +5,8 @@
 cp2k/8.2 https://github.com/cp2k/cp2k/releases/download/v8.2.0/cp2k-8.2.tar.bz2
 
 [DEPENDENCY]
+set -e
+set -x
 ./jarvis -install kgcc/9.3.1 com
 module purge
 module use ./software/modulefiles
@@ -15,16 +17,19 @@ export FC=`which gfortran`
 ./jarvis -install openmpi/4.1.2 gcc
 module load openmpi4/4.1.2
 ./jarvis -install gmp/6.2.0 gcc
+module load gmp/6.2.0
 ./jarvis -install boost/1.72.0 gcc
+module load boost/1.72.0
 ./jarvis -install libint/2.6.0 gcc+mpi
 ./jarvis -install fftw/3.3.8 gcc+mpi
+./jarvis -install kml/1.4.0/gcc gcc
 ./jarvis -install openblas/0.3.18 gcc
-module load openblas/0.3.18
-./jarvis -install scalapack/2.1.0 gcc+mpi
+./jarvis -install scalapack/2.1.0/kml gcc+mpi
 ./jarvis -install spglib/1.16.0 gcc
 ./jarvis -install libxc/5.1.4 gcc
 ./jarvis -install gsl/2.6 gcc
 module load gsl/2.6
+module load openblas/0.3.18
 ./jarvis -install plumed/2.6.2 gcc+mpi
 ./jarvis -install libvori/21.04.12 gcc
 #release CP2K
@@ -32,24 +37,31 @@ tar -jxvf downloads/cp2k-8.2.tar.bz2
 
 [ENV]
 module purge
+module use ./software/modulefiles
 module load kgcc9/9.3.1
 module load openmpi4/4.1.2
+module load scalapack-kml/2.1.0
 module load gsl/2.6
 
 [APP]
 app_name = CP2K
 build_dir = ${JARVIS_ROOT}/cp2k-8.2/
-binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/local-cuda/
+binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-cuda/
 case_dir = ${JARVIS_ROOT}/cp2k-8.2/benchmarks/QS/
 
 [BUILD]
-make -j 128 ARCH=local-cuda VERSION=psmp
+cd ./arch
+cp ${JARVIS_ROOT}/templates/CP2K/8.2/Linux-arm-cuda.psmp ./
+sed -i "11s%path1%${JARVIS_ROOT}/software/libs/kgcc9/openmpi4%g" Linux-arm-cuda.psmp
+sed -i "12s%path2%${JARVIS_ROOT}/software/libs/kgcc9%g" Linux-arm-cuda.psmp
+cd ..
+make -j 128 ARCH=Linux-arm-cuda VERSION=psmp
 
 [CLEAN]
-make -j 128 ARCH=local-cuda VERSION=psmp clean
+make -j 128 ARCH=Linux-arm-cuda VERSION=psmp clean
 
 [RUN]
-run = numactl -C 0-63  mpirun --allow-run-as-root -x CUDA_VISIBLE_DEVICES=0,1 -np 64 -x OMP_NUM_THREADS=1
+run = mpirun --allow-run-as-root -x CUDA_VISIBLE_DEVICES=0,1 -np 64 -x OMP_NUM_THREADS=1
 binary = cp2k.psmp H2O-256.inp
 nodes = 1
 
-- 
Gitee


From 126e0f90c8409d314f2710730c5d351cb91f7c7d Mon Sep 17 00:00:00 2001
From: Qiyu8 <fangchunlin@huawei.com>
Date: Wed, 3 Aug 2022 10:08:43 +0800
Subject: [PATCH 2/3] fix CP2K config issue

---
 src/dataService.py                          | 4 ++--
 templates/CP2K/8.2/Linux-arm-cuda.psmp      | 8 ++++++--
 templates/CP2K/8.2/data.CP2K.arm.gpu.config | 4 ++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/dataService.py b/src/dataService.py
index 3fe03cf..c110a4f 100644
--- a/src/dataService.py
+++ b/src/dataService.py
@@ -150,7 +150,7 @@ class DataService(object,metaclass=Singleton):
 
     def get_clean_cmd(self):
         return f'''
-set -e
+{self.get_env()}
 cd {DataService.build_dir}
 {DataService.clean_cmd}
 '''
@@ -182,4 +182,4 @@ cd {DataService.build_dir}
 {self.get_env()}
 cd {DataService.case_dir}
 {self.get_run()}
-'''
\ No newline at end of file
+'''
diff --git a/templates/CP2K/8.2/Linux-arm-cuda.psmp b/templates/CP2K/8.2/Linux-arm-cuda.psmp
index b85eaea..ea5ae66 100644
--- a/templates/CP2K/8.2/Linux-arm-cuda.psmp
+++ b/templates/CP2K/8.2/Linux-arm-cuda.psmp
@@ -1,8 +1,11 @@
 # Tested with: GFortran 7.4.0, MPICH 3.3, LAPACK 3.8.0, ScaLAPACK 2.0.2, FFTW 3.3, Libint 2.6.0, Libxc 4.3.4, libxsmm 1.14, ELPA 2019.05.001, PLUMED 2.5.2, SPGLIB 1.12.2
 # Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2019)
 NVCC    = /usr/local/cuda/bin/nvcc
-DFLAGS += -D__ACC -D__DBCSR_ACC -D__PW_CUDA
-LIBS   += -L/usr/local/cuda/lib64 -lnvToolsExt -lnvrtc -lcudart -lcublas -lcufft -lnvrtc
+NVFLAGS = -O3 -g -w --std=c++11
+OFFLOAD_TARGET = cuda
+GPUVER = V100
+DFLAGS += -D__CUDA -D__ACC -D__DBCSR_ACC -D__PW_CUDA -D__OFFLOAD_CUDA
+
 CC          = mpicc
 FC          = mpif90
 LD          = mpif90
@@ -50,3 +53,4 @@ LIBS       += -L$(MPI_PATH)/scalapack-kml/2.1.0/lib -lscalapack
 LIBS       += -L/usr/local/kml/lib/kblas/omp -lkblas
 LIBS       += -L/usr/local/kml/lib -lklapack_full
 LIBS       += -ldl -lpthread -lstdc++
+LIBS   += -L/usr/local/cuda/lib64 -lnvToolsExt -lnvrtc -lcudart -lcublas -lcufft -lnvrtc
diff --git a/templates/CP2K/8.2/data.CP2K.arm.gpu.config b/templates/CP2K/8.2/data.CP2K.arm.gpu.config
index 5a4f8c8..0c97435 100644
--- a/templates/CP2K/8.2/data.CP2K.arm.gpu.config
+++ b/templates/CP2K/8.2/data.CP2K.arm.gpu.config
@@ -52,8 +52,8 @@ case_dir = ${JARVIS_ROOT}/cp2k-8.2/benchmarks/QS/
 [BUILD]
 cd ./arch
 cp ${JARVIS_ROOT}/templates/CP2K/8.2/Linux-arm-cuda.psmp ./
-sed -i "11s%path1%${JARVIS_ROOT}/software/libs/kgcc9/openmpi4%g" Linux-arm-cuda.psmp
-sed -i "12s%path2%${JARVIS_ROOT}/software/libs/kgcc9%g" Linux-arm-cuda.psmp
+sed -i "14s%path1%${JARVIS_ROOT}/software/libs/kgcc9/openmpi4%g" Linux-arm-cuda.psmp
+sed -i "15s%path2%${JARVIS_ROOT}/software/libs/kgcc9%g" Linux-arm-cuda.psmp
 cd ..
 make -j 128 ARCH=Linux-arm-cuda VERSION=psmp
 
-- 
Gitee


From 93e5388c21cccb122a3a9fde1e063e4ee46649fa Mon Sep 17 00:00:00 2001
From: Qiyu8 <fangchunlin@huawei.com>
Date: Wed, 3 Aug 2022 15:50:18 +0800
Subject: [PATCH 3/3] add CP2k GPU config

---
 templates/CP2K/8.2/Linux-arm-cuda.psmp      | 56 --------------
 templates/CP2K/8.2/data.CP2K.arm.gpu.config | 81 ++++++++++++++++++---
 2 files changed, 72 insertions(+), 65 deletions(-)
 delete mode 100644 templates/CP2K/8.2/Linux-arm-cuda.psmp

diff --git a/templates/CP2K/8.2/Linux-arm-cuda.psmp b/templates/CP2K/8.2/Linux-arm-cuda.psmp
deleted file mode 100644
index ea5ae66..0000000
--- a/templates/CP2K/8.2/Linux-arm-cuda.psmp
+++ /dev/null
@@ -1,56 +0,0 @@
-# Tested with: GFortran 7.4.0, MPICH 3.3, LAPACK 3.8.0, ScaLAPACK 2.0.2, FFTW 3.3, Libint 2.6.0, Libxc 4.3.4, libxsmm 1.14, ELPA 2019.05.001, PLUMED 2.5.2, SPGLIB 1.12.2
-# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2019)
-NVCC    = /usr/local/cuda/bin/nvcc
-NVFLAGS = -O3 -g -w --std=c++11
-OFFLOAD_TARGET = cuda
-GPUVER = V100
-DFLAGS += -D__CUDA -D__ACC -D__DBCSR_ACC -D__PW_CUDA -D__OFFLOAD_CUDA
-
-CC          = mpicc
-FC          = mpif90
-LD          = mpif90
-AR          = ar -r
-
-MPI_PATH        = path1
-GNU_PATH        = path2
-
-include       $(MPI_PATH)/plumed/2.6.2/lib/plumed/src/lib/Plumed.inc.static
-
-FFTW_INC    = $(MPI_PATH)/fftw/3.3.8/include
-FFTW_LIB    = $(MPI_PATH)/fftw/3.3.8/lib
-
-LIBINT_INC  = $(MPI_PATH)/libint/2.6.0/include
-LIBINT_LIB  = $(MPI_PATH)/libint/2.6.0/lib
-
-LIBXC_INC   = $(GNU_PATH)/libxc/5.1.4/include
-LIBXC_LIB   = $(GNU_PATH)/libxc/5.1.4/lib
-
-SPGLIB_INC  = $(GNU_PATH)/spglib/1.16.0/include
-SPGLIB_LIB  = $(GNU_PATH)/spglib/1.16.0/lib64
-
-CFLAGS      = -O3 -g -mtune=native -fopenmp
-
-DFLAGS      = -D__FFTW3 -D__LIBINT -D__LIBXC
-DFLAGS     += -D__MPI_VERSION=3 -D__PLUMED2 -D__SPGLIB
-DFLAGS     += -D__parallel -D__SCALAPACK
-
-FCFLAGS     = $(CFLAGS) $(DFLAGS)
-FCFLAGS    += -ffree-form -ffree-line-length-none
-FCFLAGS    += -fopenmp
-FCFLAGS    += -ftree-vectorize -funroll-loops -std=f2008
-FCFLAGS    += -I$(FFTW_INC) -I$(LIBINT_INC) -I$(LIBXC_INC)
-
-LDFLAGS     = $(FCFLAGS) -static-libgfortran
-
-LIBS        = $(PLUMED_DEPENDENCIES) -L$(GNU_PATH)/gsl/2.6/lib/ -lgsl -lgslcblas -lz
-LIBS       += $(LIBXC_LIB)/libxcf03.a
-LIBS       += $(LIBXC_LIB)/libxc.a
-LIBS       += $(LIBINT_LIB)/libint2.a
-LIBS       += $(SPGLIB_LIB)/libsymspg.a
-LIBS       += $(FFTW_LIB)/libfftw3.a
-LIBS       += $(FFTW_LIB)/libfftw3_threads.a
-LIBS       += -L$(MPI_PATH)/scalapack-kml/2.1.0/lib -lscalapack
-LIBS       += -L/usr/local/kml/lib/kblas/omp -lkblas
-LIBS       += -L/usr/local/kml/lib -lklapack_full
-LIBS       += -ldl -lpthread -lstdc++
-LIBS   += -L/usr/local/cuda/lib64 -lnvToolsExt -lnvrtc -lcudart -lcublas -lcufft -lnvrtc
diff --git a/templates/CP2K/8.2/data.CP2K.arm.gpu.config b/templates/CP2K/8.2/data.CP2K.arm.gpu.config
index 0c97435..c6dbfa6 100644
--- a/templates/CP2K/8.2/data.CP2K.arm.gpu.config
+++ b/templates/CP2K/8.2/data.CP2K.arm.gpu.config
@@ -42,26 +42,89 @@ module load kgcc9/9.3.1
 module load openmpi4/4.1.2
 module load scalapack-kml/2.1.0
 module load gsl/2.6
+export CUDA_INCLUDE_DIRS=/usr/local/cuda/include
+export CUDA_CUDART_LIBRARY=/usr/local/cuda/lib64/libcudart.so
+export CUDA_HOME=/usr/local/cuda
+export PATH=$CUDA_HOME/bin:$PATH
+export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+export C_INCLUDE_PATH=$CUDA_HOME/include:$C_INCLUDE_PATH
+export LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH
 
 [APP]
 app_name = CP2K
 build_dir = ${JARVIS_ROOT}/cp2k-8.2/
-binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-cuda/
+binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/arm-cuda/
 case_dir = ${JARVIS_ROOT}/cp2k-8.2/benchmarks/QS/
 
 [BUILD]
-cd ./arch
-cp ${JARVIS_ROOT}/templates/CP2K/8.2/Linux-arm-cuda.psmp ./
-sed -i "14s%path1%${JARVIS_ROOT}/software/libs/kgcc9/openmpi4%g" Linux-arm-cuda.psmp
-sed -i "15s%path2%${JARVIS_ROOT}/software/libs/kgcc9%g" Linux-arm-cuda.psmp
-cd ..
-make -j 128 ARCH=Linux-arm-cuda VERSION=psmp
+cat >arch/arm-cuda.psmp <<"EOF"
+NVCC    = /usr/local/cuda/bin/nvcc
+NVFLAGS = -O3 -g -w --std=c++11 -arch=sm_80
+OFFLOAD_TARGET = cuda
+GPUVER = V100
+
+CC          = mpicc
+CXX         = mpic++
+FC          = mpif90
+LD          = mpif90
+AR          = ar -r
+
+MPI_PATH        = ${JARVIS_ROOT}/software/libs/kgcc9/openmpi4
+GNU_PATH        = ${JARVIS_ROOT}/software/libs/kgcc9
+
+include       $(MPI_PATH)/plumed/2.6.2/lib/plumed/src/lib/Plumed.inc.static
+
+FFTW_INC    = $(MPI_PATH)/fftw/3.3.8/include
+FFTW_LIB    = $(MPI_PATH)/fftw/3.3.8/lib
+
+LIBINT_INC  = $(MPI_PATH)/libint/2.6.0/include
+LIBINT_LIB  = $(MPI_PATH)/libint/2.6.0/lib
+
+LIBVORI_LIB = $(GNU_PATH)/libvori/21.04.12/lib
+
+LIBXC_INC   = $(GNU_PATH)/libxc/5.1.4/include
+LIBXC_LIB   = $(GNU_PATH)/libxc/5.1.4/lib
+
+SPGLIB_INC  = $(GNU_PATH)/spglib/1.16.0/include
+SPGLIB_LIB  = $(GNU_PATH)/spglib/1.16.0/lib64
+
+CFLAGS      = -O3 -g -mtune=native -fopenmp
+
+DFLAGS      = -D__FFTW3 -D__LIBINT -D__LIBXC -D__GRID_CUDA
+DFLAGS     += -D__MPI_VERSION=3 -D__PLUMED2 -D__SPGLIB
+DFLAGS     += -D__parallel -D__SCALAPACK
+DFLAGS     += -D__CUDA -D__DBCSR_ACC
+FCFLAGS     = $(CFLAGS) $(DFLAGS)
+FCFLAGS    += -ffree-form -ffree-line-length-none
+FCFLAGS    += -fopenmp
+FCFLAGS    += -ftree-vectorize -funroll-loops -std=f2008
+FCFLAGS    += -I$(FFTW_INC) -I$(LIBINT_INC) -I$(LIBXC_INC)
+
+LDFLAGS     = $(FCFLAGS) -static-libgfortran
+
+LIBS        = $(PLUMED_DEPENDENCIES) -L$(GNU_PATH)/gsl/2.6/lib/ -lgsl -lgslcblas -lz
+LIBS       += $(LIBVORI_LIB)/libvori.a
+LIBS       += $(LIBXC_LIB)/libxcf03.a
+LIBS       += $(LIBXC_LIB)/libxc.a
+LIBS       += $(LIBINT_LIB)/libint2.a
+LIBS       += $(SPGLIB_LIB)/libsymspg.a
+LIBS       += $(FFTW_LIB)/libfftw3.a
+LIBS       += $(FFTW_LIB)/libfftw3_threads.a
+LIBS       += -L$(MPI_PATH)/scalapack-kml/2.1.0/lib -lscalapack
+LIBS       += -L/usr/local/kml/lib/kblas/omp -lkblas
+LIBS       += -L/usr/local/kml/lib -lklapack_full
+LIBS       += -ldl -lpthread -lstdc++
+LIBS       += -L/usr/local/cuda/lib64 -lnvToolsExt -lnvrtc -lcudart -lcublas -lcufft -lnvrtc -lcuda
+EOF
+
+make -j 128 ARCH=arm-cuda VERSION=psmp
+ldd exe/arm-cuda/cp2k.psmp
 
 [CLEAN]
-make -j 128 ARCH=Linux-arm-cuda VERSION=psmp clean
+make -j 128 ARCH=arm-cuda VERSION=psmp realclean
 
 [RUN]
-run = mpirun --allow-run-as-root -x CUDA_VISIBLE_DEVICES=0,1 -np 64 -x OMP_NUM_THREADS=1
+run = mpirun --allow-run-as-root --mca btl ^openib -np 64 -x OMP_NUM_THREADS=1
 binary = cp2k.psmp H2O-256.inp
 nodes = 1
 
-- 
Gitee