From c934b03674e032d8849e22545e51362a441a5848 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 27 Feb 2024 15:07:36 +0800
Subject: [PATCH 01/19] update data.NWchem-new.arm.cpu.config.def

---
 .../6.8.1/data.NWchem-new.arm.cpu.config.def  | 53 +++++++++++++++++++
 .../6.8.1/data.NWchem.arm.cpu.config.def      |  9 ++++
 2 files changed, 62 insertions(+)
 create mode 100644 container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def

diff --git a/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def b/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def
new file mode 100644
index 0000000..6f5af9c
--- /dev/null
+++ b/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def
@@ -0,0 +1,53 @@
+BootStrap: library
+From: helloxusuo/repo/base:oe2203sp2 
+
+%environment
+    source /etc/profile || true
+    source /etc/profile.d/modules.sh
+    cd /hpcrunner
+    module use ./software/moduledeps/bisheng3.2.0
+    source env.sh
+
+%post
+    # Install the necessary development environment
+    #yum install -y environment-modules git dmidecode pciutils wget vim unzip curl zlib zlib-devel
+    # Install base gcc
+    #yum install -y gcc gcc-c++ gcc-gfortran glibc-devel make libgfortran cmake
+    # install network package
+    #yum install -y flex tcsh tcl lsof tk bc
+    
+    #source /etc/profile || true
+    cd /hpcrunner
+    git init
+    git config --global http.sslVerify false
+    git status
+    
+    git pull https://gitee.com/openeuler/hpcrunner
+    cd ../ 
+    mkdir linshi
+    cd linshi
+    git clone https://gitee.com/chenshaoheng/hpcrunner.git
+    cp hpcrunner/templates/nwchem/6.8.1/data.nwchem-new.arm.cpu.config /hpcrunner/templates/nwchem/6.8.1/ -f
+   
+    cd -
+    rm linshi -rf
+    cd /hpcrunner
+    source ./init.sh
+    # add your personal files here
+    
+    # Switch config
+    ./jarvis -use templates/nwchem/6.8.1/data.nwchem-new.arm.cpu.config 
+    # download dependency
+    ./jarvis -d
+    # install dependency
+    ./jarvis -dp
+    # build hpc
+    ./jarvis -b
+    # run hpc
+    ./jarvis -r
+    # clean tmp directory
+    rm -rf  downloads
+
+    
+%labels
+    Author chen
diff --git a/container/nwchem/6.8.1/data.NWchem.arm.cpu.config.def b/container/nwchem/6.8.1/data.NWchem.arm.cpu.config.def
index 817bf39..6f5af9c 100644
--- a/container/nwchem/6.8.1/data.NWchem.arm.cpu.config.def
+++ b/container/nwchem/6.8.1/data.NWchem.arm.cpu.config.def
@@ -17,11 +17,20 @@ From: helloxusuo/repo/base:oe2203sp2
     #yum install -y flex tcsh tcl lsof tk bc
     
     #source /etc/profile || true
+    cd /hpcrunner
     git init
     git config --global http.sslVerify false
     git status
     
     git pull https://gitee.com/openeuler/hpcrunner
+    cd ../ 
+    mkdir linshi
+    cd linshi
+    git clone https://gitee.com/chenshaoheng/hpcrunner.git
+    cp hpcrunner/templates/nwchem/6.8.1/data.nwchem-new.arm.cpu.config /hpcrunner/templates/nwchem/6.8.1/ -f
+   
+    cd -
+    rm linshi -rf
     cd /hpcrunner
     source ./init.sh
     # add your personal files here
-- 
Gitee


From d165dd3da069651936c3cc44f2eefd295098d07b Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 5 Mar 2024 11:38:24 +0800
Subject: [PATCH 02/19] add nwchem-sve.arm.cpu.config

---
 .../6.8.1/data.nwchem-sve.arm.cpu.config      | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
new file mode 100644
index 0000000..5826e42
--- /dev/null
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -0,0 +1,60 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+nwchem/6.8.1 https://codeload.github.com/NWChemgit/NWChem/tar.gz/6.8.1-release nwchem-6.8.1-release.tar.gz
+ga/5.6.5 $JARVIS_PROXY/edoapra/ga/releases/download/v5.6.5/ga-5.6.5.tar.gz
+
+
+[DEPENDENCY]
+module purge
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 clang
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+
+
+[ENV]
+module use ./software/modulefiles
+module purge
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+export CC=clang CXX=clang++ FC=flang
+export PMIX_INSTALL_PREFIX=${OPAL_PREFIX}
+cd /hpcrunner
+CUR_PATH=$(pwd)
+export JARVIS_ROOT=${CUR_PATH}
+export JARVIS_TMP=${CUR_PATH}/tmp
+export PATH=${JARVIS_TMP}/nwchem-6.8.1-release/bin/LINUX64:$PATH
+
+[APP]
+app_name = nwchem
+build_dir = ${JARVIS_ROOT}
+binary_dir = ${JARVIS_TMP}/nwchem-6.8.1-release/bin/LINUX64
+case_dir =
+
+[BUILD]
+cd ${JARVIS_TMP}
+tar -xvf ${JARVIS_DOWNLOAD}/nwchem-6.8.1-release.tar.gz
+cd nwchem-6.8.1-release
+cd src/tools
+tar -zxvf ${JARVIS_DOWNLOAD}/ga-5.6.5.tar.gz
+export NWCHEM_TOP=${JARVIS_TMP}/nwchem-6.8.1-release
+export NWCHEM_TARGET=LINUX64
+export NWCHEM_MODULES=all
+export USE_MPI=y
+export USE_INTERNALBLAS=y
+export NWCHEM_LONG_PATHS=Y
+cd ${JARVIS_TMP}/nwchem-6.8.1-release/src
+make nwchem_config
+make FC=flang CC=clang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
+
+[RUN]
+run =  
+binary =
+nodes = 1
+
-- 
Gitee


From 96a888b1d52a5e10b4d0d3d5cc3ff110ae2d7a10 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 5 Mar 2024 15:03:45 +0800
Subject: [PATCH 03/19] fix nwchem-sve

---
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
index 5826e42..8c92594 100644
--- a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -51,7 +51,7 @@ export USE_INTERNALBLAS=y
 export NWCHEM_LONG_PATHS=Y
 cd ${JARVIS_TMP}/nwchem-6.8.1-release/src
 make nwchem_config
-make FC=flang CC=clang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
+make FC=flang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
 
 [RUN]
 run =  
-- 
Gitee


From e5ea028d9fe4ea0c5b505220cce9d62dfdb124fa Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 5 Mar 2024 15:11:46 +0800
Subject: [PATCH 04/19] fix nwchem-sve

---
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
index 8c92594..a60befd 100644
--- a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -33,8 +33,8 @@ export PATH=${JARVIS_TMP}/nwchem-6.8.1-release/bin/LINUX64:$PATH
 
 [APP]
 app_name = nwchem
-build_dir = ${JARVIS_ROOT}
-binary_dir = ${JARVIS_TMP}/nwchem-6.8.1-release/bin/LINUX64
+build_dir = 
+binary_dir = 
 case_dir =
 
 [BUILD]
@@ -56,5 +56,5 @@ make FC=flang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true
 [RUN]
 run =  
 binary =
-nodes = 1
+nodes = 
 
-- 
Gitee


From c1e712abe7ffa88b9825fbe74af04057d4d90c35 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 5 Mar 2024 08:14:44 +0000
Subject: [PATCH 05/19] update
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config.

Signed-off-by: chenshaoheng <chenshaoheng3@h-partners.com>
---
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
index a60befd..991f086 100644
--- a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -33,8 +33,8 @@ export PATH=${JARVIS_TMP}/nwchem-6.8.1-release/bin/LINUX64:$PATH
 
 [APP]
 app_name = nwchem
-build_dir = 
-binary_dir = 
+build_dir = ${JARVIS_TMP}/nwchem-6.8.1-release
+binary_dir = ${JARVIS_TMP}/nwchem-6.8.1-release/bin/LINUX64
 case_dir =
 
 [BUILD]
@@ -56,5 +56,5 @@ make FC=flang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true
 [RUN]
 run =  
 binary =
-nodes = 
+nodes = 1
 
-- 
Gitee


From f47767d9fd3ea499ee1f7227c07f894faee2349c Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 5 Mar 2024 09:46:59 +0000
Subject: [PATCH 06/19] update
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config.

Signed-off-by: chenshaoheng <chenshaoheng3@h-partners.com>
---
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
index 991f086..8899c51 100644
--- a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -51,7 +51,7 @@ export USE_INTERNALBLAS=y
 export NWCHEM_LONG_PATHS=Y
 cd ${JARVIS_TMP}/nwchem-6.8.1-release/src
 make nwchem_config
-make FC=flang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
+make FC=flang CC=clang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
 
 [RUN]
 run =  
-- 
Gitee


From bad902fbd53659d8f60a86aa5c95ceb7c0313744 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 5 Mar 2024 12:28:00 +0000
Subject: [PATCH 07/19] update
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config.

Signed-off-by: chenshaoheng <chenshaoheng3@h-partners.com>
---
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
index 8899c51..13cd18d 100644
--- a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -51,7 +51,7 @@ export USE_INTERNALBLAS=y
 export NWCHEM_LONG_PATHS=Y
 cd ${JARVIS_TMP}/nwchem-6.8.1-release/src
 make nwchem_config
-make FC=flang CC=clang CFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
+make -j32 FC=flang CC=clang COPTIONS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FOPTIONS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
 
 [RUN]
 run =  
-- 
Gitee


From 87ce8b89948e13cb38a66b701857c44e056b61dc Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Wed, 6 Mar 2024 09:07:19 +0000
Subject: [PATCH 08/19] update
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config.

Signed-off-by: chenshaoheng <chenshaoheng3@h-partners.com>
---
 templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
index 13cd18d..1db9995 100644
--- a/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
+++ b/templates/nwchem/6.8.1/data.nwchem-sve.arm.cpu.config
@@ -51,7 +51,9 @@ export USE_INTERNALBLAS=y
 export NWCHEM_LONG_PATHS=Y
 cd ${JARVIS_TMP}/nwchem-6.8.1-release/src
 make nwchem_config
-make -j32 FC=flang CC=clang COPTIONS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" FOPTIONS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
+export COPTIONS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" 
+export FOPTIONS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true"
+make FC=flang
 
 [RUN]
 run =  
-- 
Gitee


From 176b2ab6c5331f942851abe5b146efe32dba5ccf Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Thu, 7 Mar 2024 01:17:17 +0000
Subject: [PATCH 09/19] update
 container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def.

Signed-off-by: chenshaoheng <chenshaoheng3@h-partners.com>
---
 container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def b/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def
index 6f5af9c..5f202a6 100644
--- a/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def
+++ b/container/nwchem/6.8.1/data.NWchem-new.arm.cpu.config.def
@@ -46,7 +46,7 @@ From: helloxusuo/repo/base:oe2203sp2
     # run hpc
     ./jarvis -r
     # clean tmp directory
-    rm -rf  downloads
+    rm  downloads/* tmp/nwchem-6.8.1-release/src/tools/ga-5.6.5.tar.gz -rf
 
     
 %labels
-- 
Gitee


From 8300fe6f4d473d1f76f6bc0f8fd5f4b54b005170 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 18 Mar 2024 16:23:42 +0800
Subject: [PATCH 10/19] add qe/7.3/data.qe.arm-sve.cpu.config

---
 templates/qe/7.3/data.qe.arm-sve.cpu.config | 59 +++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 templates/qe/7.3/data.qe.arm-sve.cpu.config

diff --git a/templates/qe/7.3/data.qe.arm-sve.cpu.config b/templates/qe/7.3/data.qe.arm-sve.cpu.config
new file mode 100644
index 0000000..c9de3a5
--- /dev/null
+++ b/templates/qe/7.3/data.qe.arm-sve.cpu.config
@@ -0,0 +1,59 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+qe/7.3 https://gitlab.com/QEF/q-e/-/archive/qe-7.3/q-e-qe-7.3.tar
+
+[DEPENDENCY]
+module purge 
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 bisheng
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load kml-bisheng/2.2.0
+
+
+[ENV]
+module purge
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load kml-bisheng/2.2.0
+
+export BLAS_LIBS="-L${JARVIS_ROOT}/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/lib/kblas/omp -lkblas"
+export LAPACK_LIBS="-L${JARVIS_ROOT}/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/lib -lklapack_full"
+
+cd /hpcrunner
+CUR_PATH=$(pwd)
+export JARVIS_ROOT=${CUR_PATH}
+export JARVIS_TMP=${CUR_PATH}/tmp
+export PATH=${JARVIS_ROOT}/q-e-qe-7.3-kml/bin:$PATH
+
+
+[APP]
+app_name = QE
+build_dir = ${JARVIS_TMP}/q-e-qe-7.3
+binary_dir = ${JARVIS_ROOT}/q-e-qe-7.3-kml/bin
+case_dir =
+
+[BUILD]
+cd ${JARVIS_TMP}
+tar -xf ${JARVIS_DOWNLOAD}/q-e-qe-7.3.tar
+cd q-e-qe-7.3
+
+./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3 -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" CFLAGS="-O3 -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true" --with-scalapack=no --enable-openmp --prefix=$JARVIS_ROOT/q-e-qe-7.3-kml
+sed -i 's/gfortran/flang/g' make.inc
+make -j pwall
+make install
+
+[CLEAN]
+make clean
+
+[RUN]
+run =
+binary =
+nodes = 1
-- 
Gitee


From 033823709a13d333a3b22f83a586e88a7a565caa Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 18 Mar 2024 16:35:45 +0800
Subject: [PATCH 11/19] add container-cesm.sve.config

---
 templates/CESM/2.1.3/CESM-2.1.3.patch         | 117 ++++++++++++++++++
 .../data.cesm.arm.cpu.singularity.config      |  94 ++++++++++++++
 2 files changed, 211 insertions(+)
 create mode 100644 templates/CESM/2.1.3/CESM-2.1.3.patch
 create mode 100644 templates/CESM/2.1.3/data.cesm.arm.cpu.singularity.config

diff --git a/templates/CESM/2.1.3/CESM-2.1.3.patch b/templates/CESM/2.1.3/CESM-2.1.3.patch
new file mode 100644
index 0000000..b3345d1
--- /dev/null
+++ b/templates/CESM/2.1.3/CESM-2.1.3.patch
@@ -0,0 +1,117 @@
+diff -uprN CESM-release-cesm2.1.3/cime/config/cesm/machines/config_compilers.xml CESM-release-cesm2.1.3_1/cime/config/cesm/machines/config_compilers.xml
+--- CESM-release-cesm2.1.3/cime/config/cesm/machines/config_compilers.xml	2023-08-01 11:02:27.000000000 +0800
++++ CESM-release-cesm2.1.3_1/cime/config/cesm/machines/config_compilers.xml	2024-03-11 10:38:25.077406000 +0800
+@@ -1273,9 +1273,9 @@ using a fortran linker.
+   </SLIBS>
+ </compiler>
+ 
+-<compiler COMPILER="gnu" MACH="openeuler">
++<compiler MACH="openeuler" COMPILER="gnu">
+   <CFLAGS>
+-    <base> -std=gnu99 -O3 -mcpu=tsv110 </base>
++    <base> -std=gnu99 -O3 -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true </base>
+     <append compile_threaded="true"> -fopenmp </append>
+     <append DEBUG="TRUE">  -mllvm -alias-set-saturation-threshold=10000 </append>
+     <append DEBUG="FALSE">  -mllvm -alias-set-saturation-threshold=10000 </append>
+@@ -1291,7 +1291,7 @@ using a fortran linker.
+   <FFLAGS>
+     <!-- -ffree-line-length-none and -ffixed-line-length-none need to be in FFLAGS rather than in FIXEDFLAGS/FREEFLAGS
+        so that these are passed to cmake builds (cmake builds don't use FIXEDFLAGS and FREEFLAGS). -->
+-    <base> -mno-fcmla -fconvert=big-endian -O3 -mcpu=tsv110 -Hx,54,0x08  -Hx,54,0x800 -Mx,54,0x1000 -mllvm -alias-set-saturation-threshold=10000 -lstringlib -Wl,--wrap=memcpy -Wl,--wrap=memset -I/workspace/public/software/libs/netcdf/netcdf_c_4.7.4_fortran_4.5.3/bisheng2.5.0_hmpi1.2.0/include -I/workspace/public/software/libs/pnetcdf/1.12.1/bisheng2.5.0_hmpi1.2.0/include -I/workspace/public/software/libs/hdf5/1.12.0/bisheng2.5.0_hmpi1.2.0/include -I/workspace/public/software/libs/kml/1.6.0/include -I./ -I../ </base>
++    <base> -mno-fcmla -fconvert=big-endian -O3 -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true -Hx,54,0x08  -Hx,54,0x800 -Mx,54,0x1000 -mllvm -alias-set-saturation-threshold=10000 -lstringlib -Wl,--wrap=memcpy -Wl,--wrap=memset -I/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/netcdf-clang/4.8.1/include -I/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/pnetcdf/1.12.1/include -I/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/hdf5-clang/1.12.0/include -I/hpcrunner/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/include -I./ -I../ </base>
+     <append compile_threaded="true"> -fopenmp </append>
+     <!-- Ideally, we would also have 'invalid' in the ffpe-trap list. But at
+          least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and
+@@ -1312,7 +1312,7 @@ using a fortran linker.
+   </FREEFLAGS>
+   <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
+   <LDFLAGS>
+-    <append compile_threaded="true"> -fopenmp  -L/workspace/public/software/libs/netcdf/netcdf_c_4.7.4_fortran_4.5.3/bisheng2.5.0_hmpi1.2.0/lib -L/workspace/public/software/libs/pnetcdf/1.12.1/bisheng2.5.0_hmpi1.2.0/lib -L/workspace/public/software/libs/hdf5/1.12.0/bisheng2.5.0_hmpi1.2.0/lib -L/workspace/public/software/libs/kml/1.7.0_lapack3.10.1_scalapack2.2.0/lib -L/workspace/public/software/libs/kml/1.7.0_lapack3.10.1_scalapack2.2.0/lib/kblas/omp -lkblas -lklapack_full -lnetcdff -lnetcdf -lpnetcdf -lhdf5_hl -lhdf5 -lm -lz </append>
++    <append compile_threaded="true"> -fopenmp  -L/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/netcdf-clang/4.8.1/lib -L/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/pnetcdf/1.12.1/lib -L/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/hdf5-clang/1.12.0/lib -L/hpcrunner/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/lib -L/hpcrunner/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/lib/kblas/omp -lkblas -lklapack_full -lnetcdff -lnetcdf -lpnetcdf -lhdf5_hl -lhdf5 -lm -lz </append>
+   </LDFLAGS>
+   <MPICC> mpicc  </MPICC>
+   <MPICXX> mpicxx </MPICXX>
+@@ -1322,7 +1322,7 @@ using a fortran linker.
+   <SFC> flang </SFC>
+   <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+   <SLIBS>
+-      <append> -L/workspace/public/software/libs/netcdf/netcdf_c_4.7.4_fortran_4.5.3/bisheng2.5.0_hmpi1.2.0/lib -L/workspace/public/software/libs/pnetcdf/1.12.1/bisheng2.5.0_hmpi1.2.0/lib -L/workspace/public/software/libs/hdf5/1.12.0/bisheng2.5.0_hmpi1.2.0/lib -L/workspace/public/software/libs/kml/1.7.0_lapack3.10.1_scalapack2.2.0/lib -L/workspace/public/software/libs/kml/1.7.0_lapack3.10.1_scalapack2.2.0/lib/kblas/omp -lkblas -lklapack_full -lnetcdff -lnetcdf -lpnetcdf -lhdf5_hl -lhdf5 -lm -lz </append>
++      <append> -L/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/netcdf-clang/4.8.1/lib -L/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/pnetcdf/1.12.1/lib -L/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/hdf5-clang/1.12.0/lib -L/hpcrunner/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/lib -L/hpcrunner/software/libs/bisheng3.2.0/kml-bisheng/2.2.0/lib/kblas/omp -lkblas -lklapack_full -lnetcdff -lnetcdf -lpnetcdf -lhdf5_hl -lhdf5 -lm -lz </append>
+   </SLIBS>
+ </compiler>
+ 
+diff -uprN CESM-release-cesm2.1.3/cime/config/cesm/machines/config_machines.xml CESM-release-cesm2.1.3_1/cime/config/cesm/machines/config_machines.xml
+--- CESM-release-cesm2.1.3/cime/config/cesm/machines/config_machines.xml	2023-07-07 16:36:49.000000000 +0800
++++ CESM-release-cesm2.1.3_1/cime/config/cesm/machines/config_machines.xml	2024-03-11 14:46:00.188891000 +0800
+@@ -2499,21 +2499,21 @@ This allows using a different mpirun com
+       Example port to centos7 linux system with gcc, netcdf, pnetcdf and mpich
+       using modules from http://www.admin-magazine.com/HPC/Articles/Environment-Modules
+     </DESC>
+-    <NODENAME_REGEX>DC5-19-002</NODENAME_REGEX>
++    <NODENAME_REGEX>DC5-19-004</NODENAME_REGEX>
+     <OS>LINUX</OS>
+     <PROXY> https://howto.get.out </PROXY>
+     <COMPILERS>gnu</COMPILERS>
+     <MPILIBS>openmpi</MPILIBS>
+     <PROJECT>none</PROJECT>
+     <SAVE_TIMING_DIR> </SAVE_TIMING_DIR>
+-    <CIME_OUTPUT_ROOT>/workspace/home/jcyz/tezhandui/cesm/scratch</CIME_OUTPUT_ROOT>
+-    <DIN_LOC_ROOT>/workspace/home/jcyz/tezhandui/cesm/inputdata</DIN_LOC_ROOT>
+-    <DIN_LOC_ROOT_CLMFORC>/workspace/home/jcyz/tezhandui/cesminputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
+-    <DOUT_S_ROOT>/workspace/home/jcyz/tezhandui/cesm/archive/$CASE</DOUT_S_ROOT>
+-    <BASELINE_ROOT>/workspace/home/jcyz/tezhandui/cesm/cesm_baselines</BASELINE_ROOT>
+-    <CCSM_CPRNC>/workspace/home/jcyz/tezhandui/cesm/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
++    <CIME_OUTPUT_ROOT>/hpcrunner/CESM-release-cesm2.1.3/cesm/scratch</CIME_OUTPUT_ROOT>
++    <DIN_LOC_ROOT>/hpcrunner/CESM-release-cesm2.1.3/cesm/inputdata</DIN_LOC_ROOT>
++    <DIN_LOC_ROOT_CLMFORC>/hpcrunner/CESM-release-cesm2.1.3/cesm/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
++    <DOUT_S_ROOT>/hpcrunner/CESM-release-cesm2.1.3/cesm/archive/$CASE</DOUT_S_ROOT>
++    <BASELINE_ROOT>/hpcrunner/CESM-release-cesm2.1.3/cesm/cesm_baselines</BASELINE_ROOT>
++    <CCSM_CPRNC>/hpcrunner/CESM-release-cesm2.1.3/cesm/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
+     <GMAKE>make</GMAKE>
+-    <GMAKE_J>8</GMAKE_J>
++    <GMAKE_J>32</GMAKE_J>
+     <BATCH_SYSTEM>none</BATCH_SYSTEM>
+     <SUPPORTED_BY>me@my.address</SUPPORTED_BY>
+     <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
+@@ -2522,7 +2522,7 @@ This allows using a different mpirun com
+     <mpirun mpilib="default">
+       <executable>mpirun</executable>
+       <arguments>
+-        <arg name="ntasks"> --allow-run-as-root -x PATH -x LD_LIBRARY_PATH -np {{ total_tasks }}  $CCS_MPI_OPTIONS  </arg>
++        <arg name="ntasks"> --allow-run-as-root --mca btl ^openib -np {{ total_tasks }} </arg>
+       </arguments>
+     </mpirun>
+     <module_system type="module" allow_error="true">
+@@ -2537,21 +2537,20 @@ This allows using a different mpirun com
+       <modules>
+         <command name="purge"/>
+       </modules>
+-      <modules compiler="gnu">	
+-	<command name="load">/workspace/public/software/modules/compilers/bisheng/2.5.0/bisheng2.5.0</command>
+-	<command name="load">/workspace/public/software/modules/mpi/hmpi/1.2.0/bisheng2.5.0</command>
+-	<command name="load">/workspace/public/software/modules/libs/hdf5/1.12.0/bisheng2.5.0_hmpi1.2.0</command>
+-	<command name="load">/workspace/public/software/modules/libs/netcdf/netcdf_c_4.7.4_fortran_4.5.3/bisheng2.5.0_hmpi1.2.0</command>
+-	<command name="load">/workspace/public/software/modules/libs/pnetcdf/1.12.1/bisheng2.5.0_hmpi1.2.0</command>
+-	<command name="load">/workspace/public/software/modules/libs/kml/1.7.0/kml_1.7.0_lapack3.10.1_scalapack2.2.0_bisheng2.4.0</command>
++      <modules compiler="gnu">
++        <command name="load">/hpcrunner/software/modulefiles/bisheng/3.2.0-mini</command>
++        <command name="load">/hpcrunner/software/moduledeps/bisheng3.2.0/hmpi/2.3.0</command>
++        <command name="load">/hpcrunner/software/moduledeps/bisheng3.2.0-hmpi2.3.0/netcdf-clang/4.8.1</command>
++        <command name="load">/hpcrunner/software/moduledeps/bisheng3.2.0-hmpi2.3.0/pnetcdf/1.12.1</command>
++        <command name="load">/hpcrunner/software/moduledeps/bisheng3.2.0-hmpi2.3.0/hdf5-clang/1.12.0</command>
++        <command name="load">/hpcrunner/software/moduledeps/bisheng3.2.0/kml-bisheng/2.2.0</command>
+       </modules>
+     </module_system>
+     <environment_variables>
+-	    <env name="OMP_STACKSIZE">256M</env>
+-	    <env name="NETCDF_PATH">/workspace/public/software/libs/netcdf/netcdf_c_4.7.4_fortran_4.5.3/bisheng2.5.0_hmpi1.2.0</env>
+-	    <env name="HDF5_PATH">/workspace/public/software/libs/hdf5/1.12.0/bisheng2.5.0_hmpi1.2.0</env>
+-	    <env name="PNETCDF_PATH">/workspace/public/software/libs/pnetcdf/1.12.1/bisheng2.5.0_hmpi1.2.0</env>
+-	    <env name="KML_PATH">/workspace/public/software/libs/kml/1.7.0_lapack3.10.1_scalapack2.2.0</env>
++      <env name="OMP_STACKSIZE">256M</env>
++      <env name="NETCDF_PATH">/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/netcdf-clang/4.8.1</env>
++      <env name="HDF5_PATH">/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/hdf5-clang/1.12.0</env>
++      <env name="PNETCDF_PATH">/hpcrunner/software/libs/bisheng3.2.0/hmpi2.3.0/pnetcdf/1.12.1</env>
+     </environment_variables>
+     <resource_limits>
+       <resource name="RLIMIT_STACK">-1</resource>
\ No newline at end of file
diff --git a/templates/CESM/2.1.3/data.cesm.arm.cpu.singularity.config b/templates/CESM/2.1.3/data.cesm.arm.cpu.singularity.config
new file mode 100644
index 0000000..2f8197d
--- /dev/null
+++ b/templates/CESM/2.1.3/data.cesm.arm.cpu.singularity.config
@@ -0,0 +1,94 @@
+[SERVER]
+11.11.11.11
+
+[DEPENDENCY]
+set -x
+set -e
+. $CHECK_ROOT && yum install -y perl-XML-LibXML time.aarch64 patch
+module purge
+module use ./software/modulefiles
+./jarvis -install bisheng/3.2.0-mini com
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install openblas/0.3.18 clang
+./jarvis -install hmpi/2.3.0 bisheng
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+export CC=mpicc CXX=mpicxx FC=mpifort F77=mpifort
+./jarvis -install hdf5/1.12.0/clang clang+mpi
+module load hdf5-clang/1.12.0
+./jarvis -install pnetcdf/1.12.1 clang+mpi
+module load pnetcdf/1.12.1
+./jarvis -install netcdf/4.8.1/clang clang+mpi
+./jarvis -install kml/2.2.0/bisheng bisheng
+ln -sf /usr/bin/python3 /usr/bin/python
+
+[ENV]
+module purge
+module use /hpcrunner/software/modulefiles
+module load bisheng/3.2.0-mini
+module use /hpcrunner/software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load hdf5-clang/1.12.0
+module load pnetcdf/1.12.1
+module load netcdf-clang/4.8.1
+module load openblas/0.3.18
+module load kml-bisheng/2.2.0
+export LD_LIBRARY_PATH=$KML_BISHENG_PATH/lib/kblas/omp:$LD_LIBRARY_PATH
+export NETCDF_PATH=${NETCDF_CLANG_PATH}
+export CESM_PATH=/hpcrunner/CESM-release-cesm2.1.3
+export CIMEROOT=$CESM_PATH/cime
+export PYTHONPATH=$CIMEROOT/scripts/lib:$CIMEROOT/scripts/Tools:$PYTHONPATH
+export case_name_1=f19_g16_1node
+export case_name_2=f19_g16_2nodes
+
+[APP]
+app_name = CESM
+build_dir = $CESM_PATH
+binary_dir = 
+case_dir = $CESM_PATH/cime/scripts/$case_name_1
+
+[BUILD]
+set -e
+set -x
+cd /hpcrunner
+./jarvis -e
+source ./env.sh
+tar -zxvf /mnt/CESM-release-cesm2.1.3.src.tgz -C /hpcrunner
+tar -zxvf /mnt/cesm.inputdata.tgz -C $CESM_PATH/
+cd ${CESM_PATH}
+patch -p1 < /hpcrunner/templates/CESM/2.1.3/CESM-2.1.3.patch
+cd $CIMEROOT/scripts
+if [ ! -d $case_name_1 ]; then
+    ./create_newcase --case $case_name_1 --compset X --res f19_g16 --mach openeuler --compiler gnu
+fi
+cd ${case_name_1}
+./case.setup
+./xmlchange STOP_N=300
+./case.setup --reset
+./case.build --skip-provenance-check
+
+cd $CIMEROOT/scripts
+if [ ! -d $case_name_2 ]; then
+    ./create_newcase --case $case_name_2 --compset X --res f19_g16 --mach openeuler --compiler gnu
+fi
+cd ${case_name_2}
+./case.setup
+./xmlchange NTASKS=256,STOP_N=300
+./case.setup --reset
+./case.build --skip-provenance-check
+
+[CLEAN]
+cd $CIMEROOT/scripts/$case_name_1
+./case.build --clean-all
+cd $CIMEROOT/scripts/$case_name_2
+./case.build --clean-all
+
+[RUN]
+#run = rm -rf cesm_* && chmod +x run.sh && dsub -s run.sh
+run = ./case.submit
+binary =
+nodes = 1
+
+[BATCH]
+#! /bin/bash
\ No newline at end of file
-- 
Gitee


From 6e893293cd15f6eaf0e33b443d2259284cd08ed6 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 18 Mar 2024 17:00:04 +0800
Subject: [PATCH 12/19] add templates/gromacs/2022.5/*

---
 .../2022.5/data.gromacs.arm-sve.cpu.config    | 65 +++++++++++++++++++
 .../2022.5/data.gromacs.arm.cpu.config        | 65 +++++++++++++++++++
 2 files changed, 130 insertions(+)
 create mode 100644 templates/gromacs/2022.5/data.gromacs.arm-sve.cpu.config
 create mode 100644 templates/gromacs/2022.5/data.gromacs.arm.cpu.config

diff --git a/templates/gromacs/2022.5/data.gromacs.arm-sve.cpu.config b/templates/gromacs/2022.5/data.gromacs.arm-sve.cpu.config
new file mode 100644
index 0000000..56c4899
--- /dev/null
+++ b/templates/gromacs/2022.5/data.gromacs.arm-sve.cpu.config
@@ -0,0 +1,65 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+gromacs/2022.5 http://ftp.gromacs.org/pub/gromacs/gromacs-2022.5.tar.gz
+
+[DEPENDENCY]
+module purge
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 clang
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+./jarvis -install kml/2.2.0/bisheng bisheng
+module load kml-bisheng/2.2.0
+
+[ENV]
+module purge
+module use ./software/modulefiles
+module purge
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load kml-bisheng/2.2.0
+module load hmpi/2.3.0
+export CC=mpicc CXX=mpicxx FC=mpifort
+export PMIX_INSTALL_PREFIX=${OPAL_PREFIX}
+cd /hpcrunner
+CUR_PATH=$(pwd)
+export JARVIS_ROOT=${CUR_PATH}
+export JARVIS_TMP=${CUR_PATH}/tmp
+export PATH=${JARVIS_ROOT}/gromacs-2022.5-kml/bin:$PATH
+
+[APP]
+app_name = gromacs
+build_dir = $JARVIS_TMP/gromacs-2022.5
+binary_dir = $JARVIS_ROOT/gromacs-2022.5-kml/bin 
+case_dir =
+
+[BUILD]
+cd ${JARVIS_TMP}
+rm -rf gromacs-2022.5
+tar -xf ${JARVIS_DOWNLOAD}/gromacs-2022.5.tar.gz
+cd gromacs-2022.5
+
+sed -i '204s/return fftw_version;/return 0;/g' src/gromacs/utility/binaryinformation.cpp
+sed -i '206s/return fftwf_version;/return 0;/g' src/gromacs/utility/binaryinformation.cpp
+sed -i '505s/int fftwflags = FFTW_DESTROY_INPUT;/int fftwflags = 0;/g' src/gromacs/fft/fft5d.cpp
+rm -rf build
+mkdir -p build
+cd build
+FLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true -O3 -lgfortran"; CFLAGS=$FLAGS CXXFLAGS="-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true -O3 -lgfortran" CC=mpicc CXX=mpicxx FC=mpifort cmake -DCMAKE_INSTALL_PREFIX=$JARVIS_ROOT/gromacs-2022.5-kml -DBUILD_SHARED_LIBS=on -DBUILD_TESTING=on -DREGRESSIONTEST_DOWNLOAD=off -DGMX_BUILD_OWN_FFTW=off -DGMX_SIMD=ARM_NEON_ASIMD -DGMX_DOUBLE=off -DGMX_EXTERNAL_BLAS=on -DGMX_EXTERNAL_LAPACK=on -DGMX_FFT_LIBRARY=fftw3 -DGMX_BLAS_USER=${KML_BISHENG_PATH}/lib/kblas/omp/libkblas.so -DGMX_LAPACK_USER=${KML_BISHENG_PATH}/lib/libklapack_full.so -DFFTWF_LIBRARY=${KML_BISHENG_PATH}/lib/libkfftf.so -DFFTWF_INCLUDE_DIR=${KML_BISHENG_PATH}/include -DGMX_GPU=off -DGMX_MPI=on -DGMX_OPENMP=on -DGMX_X11=off ../
+
+make -j40 V=1
+make install
+
+[CLEAN]
+./clean_build
+
+[RUN]
+run =
+binary =
+nodes = 1
+
diff --git a/templates/gromacs/2022.5/data.gromacs.arm.cpu.config b/templates/gromacs/2022.5/data.gromacs.arm.cpu.config
new file mode 100644
index 0000000..efeeb8a
--- /dev/null
+++ b/templates/gromacs/2022.5/data.gromacs.arm.cpu.config
@@ -0,0 +1,65 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+gromacs/2022.5 http://ftp.gromacs.org/pub/gromacs/gromacs-2022.5.tar.gz
+
+[DEPENDENCY]
+module purge
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 clang
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+./jarvis -install kml/2.2.0/bisheng bisheng
+module load kml-bisheng/2.2.0
+
+[ENV]
+module purge
+module use ./software/modulefiles
+module purge
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load kml-bisheng/2.2.0
+module load hmpi/2.3.0
+export CC=mpicc CXX=mpicxx FC=mpifort
+export PMIX_INSTALL_PREFIX=${OPAL_PREFIX}
+cd /hpcrunner
+CUR_PATH=$(pwd)
+export JARVIS_ROOT=${CUR_PATH}
+export JARVIS_TMP=${CUR_PATH}/tmp
+export PATH=${JARVIS_ROOT}/gromacs-2022.5-kml/bin:$PATH
+
+[APP]
+app_name = gromacs
+build_dir = $JARVIS_TMP/gromacs-2022.5
+binary_dir = $JARVIS_ROOT/gromacs-2022.5-kml/bin 
+case_dir =
+
+[BUILD]
+cd ${JARVIS_TMP}
+rm -rf gromacs-2022.5
+tar -xf ${JARVIS_DOWNLOAD}/gromacs-2022.5.tar.gz
+cd gromacs-2022.5
+
+sed -i '204s/return fftw_version;/return 0;/g' src/gromacs/utility/binaryinformation.cpp
+sed -i '206s/return fftwf_version;/return 0;/g' src/gromacs/utility/binaryinformation.cpp
+sed -i '505s/int fftwflags = FFTW_DESTROY_INPUT;/int fftwflags = 0;/g' src/gromacs/fft/fft5d.cpp
+rm -rf build
+mkdir -p build
+cd build
+FLAGS="-mcpu=tsv110 -O3 -lgfortran"; CFLAGS=$FLAGS CXXFLAGS="-mcpu=tsv110 -O3 -lgfortran" CC=mpicc CXX=mpicxx FC=mpifort cmake -DCMAKE_INSTALL_PREFIX=$JARVIS_ROOT/gromacs-2022.5-kml -DBUILD_SHARED_LIBS=on -DBUILD_TESTING=on -DREGRESSIONTEST_DOWNLOAD=off -DGMX_BUILD_OWN_FFTW=off -DGMX_SIMD=ARM_NEON_ASIMD -DGMX_DOUBLE=off -DGMX_EXTERNAL_BLAS=on -DGMX_EXTERNAL_LAPACK=on -DGMX_FFT_LIBRARY=fftw3 -DGMX_BLAS_USER=${KML_BISHENG_PATH}/lib/kblas/omp/libkblas.so -DGMX_LAPACK_USER=${KML_BISHENG_PATH}/lib/libklapack_full.so -DFFTWF_LIBRARY=${KML_BISHENG_PATH}/lib/libkfftf.so -DFFTWF_INCLUDE_DIR=${KML_BISHENG_PATH}/include -DGMX_GPU=off -DGMX_MPI=on -DGMX_OPENMP=on -DGMX_X11=off ../
+
+make -j40 V=1
+make install
+
+[CLEAN]
+./clean_build
+
+[RUN]
+run =
+binary =
+nodes = 1
+
-- 
Gitee


From d50a620331748ec4fbc57b1490d03ca9a95cc300 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 18 Mar 2024 17:07:01 +0800
Subject: [PATCH 13/19] add templates/lammps/2023.8.2/

---
 .../2023.8.2/data.lammps.arm.cpu.config       | 59 +++++++++++++++++++
 .../2023.8.2/data.lammps.arm.cpu.new.config   | 59 +++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 templates/lammps/2023.8.2/data.lammps.arm.cpu.config
 create mode 100644 templates/lammps/2023.8.2/data.lammps.arm.cpu.new.config

diff --git a/templates/lammps/2023.8.2/data.lammps.arm.cpu.config b/templates/lammps/2023.8.2/data.lammps.arm.cpu.config
new file mode 100644
index 0000000..42ad62f
--- /dev/null
+++ b/templates/lammps/2023.8.2/data.lammps.arm.cpu.config
@@ -0,0 +1,59 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+lammps/2023.8.2 https://download.lammps.org/tars/lammps-2Aug2023.tar.gz
+
+[DEPENDENCY]
+module purge
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 bisheng
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+./jarvis -install fftw/3.3.10 bisheng
+module load fftw/3.3.10
+./jarvis -install kml/2.2.0/bisheng bisheng
+module load kml-bisheng/2.2.0
+
+[ENV]
+module use ./software/modulefiles
+module purge
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load fftw/3.3.10
+module load kml-bisheng/2.2.0
+export CC=clang CXX=clang++ FC=flang
+export PATH=/hpcrunner/tmp/lammps-2Aug2023/src:$PATH
+
+[APP]
+app_name = lammps
+build_dir = ${JARVIS_ROOT}
+binary_dir = ./software/apps/bisheng/lammps/2023.8.2
+case_dir =
+
+[BUILD]
+cd ${JARVIS_ROOT}/tmp
+tar -xvf ${JARVIS_DOWNLOAD}/lammps-2Aug2023.tar.gz
+cd lammps-2Aug2023/src
+sed -i '10c\CCFLAGS = -g -O3 -fopenmp -I${JARVIS_ROOT}/software/compiler/bisheng/3.2.0/include/c++/v1 -L${JARVIS_ROOT}/software/compiler/bisheng/3.2.0/lib -std=c++11 -stdlib=libc++ -lc++ -lc++abi'   MAKE/Makefile.mpi
+sed -i '15c\LINKFLAGS =  -g -O3 -fopenmp -L${JARVIS_ROOT}/software/compiler/bisheng/3.2.0/lib -std=c++11 -stdlib=libc++ -lc++ -lc++abi'   MAKE/Makefile.mpi
+sed -i '53c\FFT_INC = -DFFT_FFTW -I${JARVIS_ROOT}/software/libs/bisheng3.2.0/fftw/3.3.10/include ' MAKE/Makefile.mpi
+sed -i '54c\FFT_PATH = -L${JARVIS_ROOT}/software/libs/bisheng3.2.0/fftw/3.3.10/lib'  MAKE/Makefile.mpi
+sed -i '55c\FFT_LIB = -lfftw3_omp -lfftw3' MAKE/Makefile.mpi
+make yes-KSPACE
+make yes-MPIIO
+make yes-MOLECULE
+make yes-MANYBODY
+make mpi -j
+
+
+
+[RUN]
+run =
+binary =
+nodes = 1
+
diff --git a/templates/lammps/2023.8.2/data.lammps.arm.cpu.new.config b/templates/lammps/2023.8.2/data.lammps.arm.cpu.new.config
new file mode 100644
index 0000000..9253347
--- /dev/null
+++ b/templates/lammps/2023.8.2/data.lammps.arm.cpu.new.config
@@ -0,0 +1,59 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+lammps/2023.8.2 https://download.lammps.org/tars/lammps-2Aug2023.tar.gz
+
+[DEPENDENCY]
+module purge
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 bisheng
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+./jarvis -install fftw/3.3.10 bisheng
+module load fftw/3.3.10
+./jarvis -install kml/2.2.0/bisheng bisheng
+module load kml-bisheng/2.2.0
+
+[ENV]
+module use ./software/modulefiles
+module purge
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load fftw/3.3.10
+module load kml-bisheng/2.2.0
+export CC=clang CXX=clang++ FC=flang
+export PATH=/hpcrunner/tmp/lammps-2Aug2023/src:$PATH
+
+[APP]
+app_name = lammps
+build_dir = ${JARVIS_ROOT}
+binary_dir = ./software/apps/bisheng/lammps/2023.8.2
+case_dir =
+
+[BUILD]
+cd ${JARVIS_ROOT}/tmp
+tar -xvf ${JARVIS_DOWNLOAD}/lammps-2Aug2023.tar.gz
+cd lammps-2Aug2023/src
+sed -i '10c\CCFLAGS = -g -O3 -fopenmp -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true -I${JARVIS_ROOT}/software/compiler/bisheng/3.2.0/include/c++/v1 -L${JARVIS_ROOT}/software/compiler/bisheng/3.2.0/lib -std=c++11 -stdlib=libc++ -lc++ -lc++abi'   MAKE/Makefile.mpi
+sed -i '15c\LINKFLAGS =  -g -O3 -fopenmp -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true -L${JARVIS_ROOT}/software/compiler/bisheng/3.2.0/lib -std=c++11 -stdlib=libc++ -lc++ -lc++abi'   MAKE/Makefile.mpi
+sed -i '53c\FFT_INC = -DFFT_FFTW -I${JARVIS_ROOT}/software/libs/bisheng3.2.0/fftw/3.3.10/include ' MAKE/Makefile.mpi
+sed -i '54c\FFT_PATH = -L${JARVIS_ROOT}/software/libs/bisheng3.2.0/fftw/3.3.10/lib'  MAKE/Makefile.mpi
+sed -i '55c\FFT_LIB = -lfftw3_omp -lfftw3' MAKE/Makefile.mpi
+make yes-KSPACE
+make yes-MPIIO
+make yes-MOLECULE
+make yes-MANYBODY
+make mpi -j
+
+
+
+[RUN]
+run =
+binary =
+nodes = 1
+
-- 
Gitee


From 24cf302918034a99b786da1e4ec7838e79a61294 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 18 Mar 2024 17:29:12 +0800
Subject: [PATCH 14/19] add templates/CP2K/8.2/

---
 templates/CP2K/8.2/Linux-arm-clang-sve.psmp   |  60 ++++++++++
 .../CP2K/8.2/data.CP2K.arm-new.cpu.config     | 111 ++++++++++++++++++
 2 files changed, 171 insertions(+)
 create mode 100644 templates/CP2K/8.2/Linux-arm-clang-sve.psmp
 create mode 100644 templates/CP2K/8.2/data.CP2K.arm-new.cpu.config

diff --git a/templates/CP2K/8.2/Linux-arm-clang-sve.psmp b/templates/CP2K/8.2/Linux-arm-clang-sve.psmp
new file mode 100644
index 0000000..e1b8650
--- /dev/null
+++ b/templates/CP2K/8.2/Linux-arm-clang-sve.psmp
@@ -0,0 +1,60 @@
+# Tested with: GFortran 7.4.0, MPICH 3.3, LAPACK 3.8.0, ScaLAPACK 2.0.2, FFTW 3.3, Libint 2.6.0, Libxc 4.3.4, libxsmm 1.14, ELPA 2019.05.001, PLUMED 2.5.2, SPGLIB 1.12.2
+# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2019)
+
+CC          = mpicc
+FC          = mpif90
+LD          = mpif90
+AR          = ar -r
+
+MPI_PATH        = path1
+GNU_PATH        = path2
+
+include       $(MPI_PATH)/plumed/2.6.2/lib/plumed/src/lib/Plumed.inc.static
+
+ELPA_VER    = 2019.05.001
+ELPA_INC    = $(MPI_PATH)/elpa/$(ELPA_VER)/include/elpa_openmp-$(ELPA_VER)
+ELPA_LIB    = $(MPI_PATH)/elpa/$(ELPA_VER)/lib
+
+FFTW_INC    = $(MPI_PATH)/fftw/3.3.8/include
+FFTW_LIB    = $(MPI_PATH)/fftw/3.3.8/lib
+
+LIBINT_INC  = $(MPI_PATH)/libint/2.6.0/include
+LIBINT_LIB  = $(MPI_PATH)/libint/2.6.0/lib
+
+LIBXC_INC   = $(GNU_PATH)/libxc/4.3.4/include
+LIBXC_LIB   = $(GNU_PATH)/libxc/4.3.4/lib
+
+SPGLIB_INC  = $(GNU_PATH)/spglib/1.16.0/include
+SPGLIB_LIB  = $(GNU_PATH)/spglib/1.16.0/lib64
+
+#LIBXSMM_INC = $(GNU_PATH)/libxsmm/1.14/include
+#LIBXSMM_LIB = $(GNU_PATH)/libxsmm/1.14/lib
+
+CFLAGS      = -O2 -g -mtune=native -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true
+
+DFLAGS      = -D__FFTW3 -D__LIBINT -D__LIBXC -D__ELPA  #-D__LIBXSMM -D__ELPA
+DFLAGS     += -D__MPI_VERSION=3 -D__PLUMED2 -D__SPGLIB
+DFLAGS     += -D__parallel -D__SCALAPACK
+
+FCFLAGS     = $(CFLAGS) $(DFLAGS)
+FCFLAGS    += -ffree-form -ffree-line-length-none
+FCFLAGS    += -fopenmp
+FCFLAGS    += -ftree-vectorize -funroll-loops -std=f2008
+FCFLAGS    += -I$(ELPA_INC)/elpa -I$(ELPA_INC)/modules
+FCFLAGS    += -I$(FFTW_INC) -I$(LIBINT_INC) -I$(LIBXC_INC) #-I$(LIBXSMM_INC)
+
+LDFLAGS     = $(FCFLAGS) -static-libgfortran
+
+LIBS        = $(PLUMED_DEPENDENCIES) -L$(GNU_PATH)/gsl/2.6/lib/ -lgsl -lgslcblas -lz
+LIBS       += $(ELPA_LIB)/libelpa_openmp.a
+LIBS       += $(LIBXC_LIB)/libxcf03.a
+LIBS       += $(LIBXC_LIB)/libxc.a
+LIBS       += $(LIBINT_LIB)/libint2.a
+LIBS       += $(SPGLIB_LIB)/libsymspg.a
+LIBS       += $(FFTW_LIB)/libfftw3.a
+LIBS       += $(FFTW_LIB)/libfftw3_threads.a
+LIBS       += $(MPI_PATH)/scalapack-lapack-bisheng/2.1.0/lib/libscalapack.a
+LIBS       += $(GNU_PATH)/lapack-bisheng/3.8.0/lib/liblapack.a
+LIBS       += $(GNU_PATH)/lapack-bisheng/3.8.0/lib/librefblas.a
+LIBS       += -ldl -lpthread -lstdc++
+
diff --git a/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config b/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config
new file mode 100644
index 0000000..8eaa92f
--- /dev/null
+++ b/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config
@@ -0,0 +1,111 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+cp2k/8.2 https://521github.com/extdomains/github.com/cp2k/cp2k/releases/download/v8.2.0/cp2k-8.2.tar.bz2 
+
+[DEPENDENCY]
+module purge
+module use ./software/modulefiles
+./jarvis -install bisheng/3.2.0-mini com
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install cmake/3.23.1 any
+module load cmake/3.23.1
+./jarvis -install hmpi/2.3.0 clang
+./jarvis -install gmp/6.2.0 clang
+./jarvis -install boost/1.72.0 clang
+./jarvis -install openblas/0.3.18 clang
+./jarvis -install spglib/1.16.0 clang
+./jarvis -install libxc/5.1.4 clang
+./jarvis -install gsl/2.6 clang
+./jarvis -install libvori/21.04.12 clang
+./jarvis -install lapack/3.8.0/bisheng clang
+
+module use ./software/moduledeps/bisheng3.2.0
+module load openblas/0.3.18
+module load gsl/2.6
+module load hmpi/2.3.0
+module load gmp/6.2.0
+module load boost/1.72.0 
+module load lapack-bisheng/3.8.0
+
+module use ./software/moduledeps/bisheng3.2.0-hmpi2.3.0
+
+export CC=mpicc CXX=mpicxx FC=mpifort
+./jarvis -install libint/2.6.0 clang+mpi
+./jarvis -install fftw/3.3.8 clang+mpi
+./jarvis -install scalapack/2.1.0/clang clang+mpi
+./jarvis -install plumed/2.6.2 clang+mpi
+module load scalapack-clang/2.1.0 openblas/0.3.18
+./jarvis -install elpa/2020.11.001 clang+mpi
+
+rm -rf cp2k-8.2
+tar -jxvf $JARVIS_DOWNLOAD/cp2k-8.2.tar.bz2
+
+[ENV]
+module purge
+module use ./software/modulefiles
+module use ./software/moduledeps/bisheng3.2.0
+module use ./software/moduledeps/bisheng3.2.0-hmpi2.3.0
+module load bisheng/3.2.0-mini
+module load boost/1.72.0  gmp/6.2.0  gsl/2.6  hmpi/2.3.0  lapack-bisheng/3.8.0  libvori/21.04.12  libxc/5.1.4  openblas/0.3.18  spglib/1.16.0
+export CC=mpicc CXX=mpicxx FC=mpifort
+module load elpa/2020.11.001  fftw/3.3.8  libint/2.6.0  plumed/2.6.2  scalapack-clang/2.1.0
+export PATH=${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-clang:$PATH
+
+[APP]
+app_name = CP2K
+build_dir = ${JARVIS_ROOT}
+binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-clang/
+case_dir = ${JARVIS_ROOT}/cp2k-8.2/benchmarks/QS/
+
+[BUILD]
+cd ${JARVIS_ROOT}/cp2k-8.2/arch
+cp ${JARVIS_ROOT}/templates/CP2K/8.2/Linux-arm-clang-sve.psmp ./
+sed -i "9s%path1%${JARVIS_ROOT}/software/libs/bisheng3.2.0/hmpi2.3.0%g" Linux-arm-clang-sve.psmp
+sed -i "10s%path2%${JARVIS_ROOT}/software/libs/bisheng3.2.0%g" Linux-arm-clang-sve.psmp
+cd ..
+make -j ARCH=Linux-arm-clang-sve VERSION=psmp
+
+[CLEAN]
+make -j 128 ARCH=Linux-arm-clang-sve VERSION=psmp clean
+
+[RUN]
+run = numactl -C 0-63  mpirun --allow-run-as-root -np 64 -map-by ppr:64:node:pe=1 -bind-to core -x OMP_NUM_THREADS=1 
+binary = cp2k.psmp H2O-256.inp
+nodes = 1
+
+[BATCH]
+#!/bin/bash
+
+logfile=cp2k.H2O-256.inp.log
+
+nvidia-smi -pm 1
+nvidia-smi -ac 1215,1410
+
+echo 3 > /proc/sys/vm/drop_caches
+echo "===run 32C*GPU===" >> $logfile
+mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+
+echo 3 > /proc/sys/vm/drop_caches
+echo "===run 32C*2GPU===" >> $logfile
+mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+
+
+echo 3 > /proc/sys/vm/drop_caches
+echo "===run 64C*GPU===" >> $logfile
+mpirun -np 64 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+
+echo 3 > /proc/sys/vm/drop_caches
+echo "===run 64C*2GPU===" >> $logfile
+mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+
+
+echo 3 > /proc/sys/vm/drop_caches
+echo "===run 128C*GPU===" >> $logfile
+mpirun -np 128 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+
+echo 3 > /proc/sys/vm/drop_caches
+echo "===run 128C*2GPU===" >> $logfile
+mpirun -np 128 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
-- 
Gitee


From 8a735cf383353bc7088b255020dfc673ec342b5f Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 18 Mar 2024 18:24:08 +0800
Subject: [PATCH 15/19] add templates/roms/4.0/*

---
 templates/roms/4.0/Linux-flang.mk             |  12 +-
 templates/roms/4.0/build_roms.sh              | 292 ++++++++++++++
 templates/roms/4.0/my_build_paths.sh          | 375 ++++++++++++++++++
 ...oms.singularity.arm.bisheng-sve.cpu.config |  81 ++++
 .../roms.singularity.arm.bisheng.cpu.config   |  79 ++++
 5 files changed, 833 insertions(+), 6 deletions(-)
 create mode 100755 templates/roms/4.0/build_roms.sh
 create mode 100755 templates/roms/4.0/my_build_paths.sh
 create mode 100644 templates/roms/4.0/roms.singularity.arm.bisheng-sve.cpu.config
 create mode 100644 templates/roms/4.0/roms.singularity.arm.bisheng.cpu.config

diff --git a/templates/roms/4.0/Linux-flang.mk b/templates/roms/4.0/Linux-flang.mk
index fd9cb14..6b1cc59 100644
--- a/templates/roms/4.0/Linux-flang.mk
+++ b/templates/roms/4.0/Linux-flang.mk
@@ -29,7 +29,7 @@
 # First the defaults
 #
                FC := mpifort
-           FFLAGS := -frepack-arrays -fvectorize -funroll-loops -mllvm -unroll-indirect-loads=true -mcpu=tsv110 -mllvm -prefetch-loop-depth=3 -mllvm -min-prefetch-stride=4 -mllvm -prefetch-distance=1600 -fstack-arrays -lstringlib -Wl,--wrap=memset -Wl,--wrap=memcpy -ljemalloc
+           FFLAGS := -mcpu=tsv110 -Wl, -O3 -w
        FIXEDFLAGS := -ffixed-form
         FREEFLAGS := -ffree-form -ffree-line-length-none
               CPP := /usr/bin/cpp
@@ -40,7 +40,7 @@
              LIBS := $(SCRATCH_DIR)/libNLM.a         # cyclic dependencies
        MOD_SUFFIX := mod
                LD := $(FC)
-          LDFLAGS := -lstringlib -Wl,--wrap=memset -Wl,--wrap=memcpy -ljemalloc
+          LDFLAGS :=
                AR := ar
           ARFLAGS := -r
             MKDIR := mkdir -p
@@ -170,7 +170,7 @@ ifdef USE_SCORPIO
 endif
 
 ifdef USE_NETCDF4
-        NF_CONFIG ?= nf-config
+        NF_CONFIG ?= $(NETCDF_CLANG_PATH)/bin/nf-config
     NETCDF_INCDIR ?= $(shell $(NF_CONFIG) --prefix)/include
              LIBS += $(shell $(NF_CONFIG) --flibs)
            INCDIR += $(NETCDF_INCDIR) $(INCDIR)
@@ -185,8 +185,8 @@ endif
 ifdef USE_HDF5
 #     HDF5_INCDIR ?= /opt/gfortransoft/serial/hdf5/include
 #     HDF5_LIBDIR ?= /opt/gfortransoft/serial/hdf5/lib
-      HDF5_INCDIR ?= $(HDF5_INC)
-      HDF5_LIBDIR ?= $(HDF5_LIBDIR)
+      HDF5_INCDIR ?= $(HDF5_CLANG_PATH)/include
+      HDF5_LIBDIR ?= $(HDF5_CLANG_PATH)/lib
         HDF5_LIBS ?= -lhdf5_fortran -lhdf5hl_fortran -lhdf5 -lz
              LIBS += -L$(HDF5_LIBDIR) $(HDF5_LIBS)
            INCDIR += $(HDF5_INCDIR)
@@ -204,7 +204,7 @@ endif
 ifdef USE_MPI
          CPPFLAGS += -DMPI
  ifdef USE_MPIF90
-               FC := mpif90
+               FC := mpifort
  else
              LIBS += -lfmpi -lmpi
  endif
diff --git a/templates/roms/4.0/build_roms.sh b/templates/roms/4.0/build_roms.sh
new file mode 100755
index 0000000..c0fffb2
--- /dev/null
+++ b/templates/roms/4.0/build_roms.sh
@@ -0,0 +1,292 @@
+#!/bin/bash
+#
+# svn $Id: build_roms.sh 1064 2021-05-10 19:55:56Z arango $
+#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+# Copyright (c) 2002-2021 The ROMS/TOMS Group                           :::
+#   Licensed under a MIT/X style license                                :::
+#   See License_ROMS.txt                                                :::
+#::::::::::::::::::::::::::::::::::::::::::::::::::::: Hernan G. Arango :::
+#                                                                       :::
+# ROMS Compiling BASH Script                                            :::
+#                                                                       :::
+# Script to compile an user application where the application-specific  :::
+# files are kept separate from the ROMS source code.                    :::
+#                                                                       :::
+# Q: How/why does this script work?                                     :::
+#                                                                       :::
+# A: The ROMS makefile configures user-defined options with a set of    :::
+#    flags such as ROMS_APPLICATION. Browse the makefile to see these.  :::
+#    If an option in the makefile uses the syntax ?= in setting the     :::
+#    default, this means that make will check whether an environment    :::
+#    variable by that name is set in the shell that calls make. If so   :::
+#    the environment variable value overrides the default (and the      :::
+#    user need not maintain separate makefiles, or frequently edit      :::
+#    the makefile, to run separate applications).                       :::
+#                                                                       :::
+# Usage:                                                                :::
+#                                                                       :::
+#    ./build_roms.sh [options]                                          :::
+#                                                                       :::
+# Options:                                                              :::
+#                                                                       :::
+#    -j [N]      Compile in parallel using N CPUs                       :::
+#                  omit argument for all available CPUs                 :::
+#                                                                       :::
+#    -p macro    Prints any Makefile macro value. For example,          :::
+#                                                                       :::
+#                  build_roms.sh -p FFLAGS                              :::
+#                                                                       :::
+#    -noclean    Do not clean already compiled objects                  :::
+#                                                                       :::
+# Notice that sometimes the parallel compilation fail to find MPI       :::
+# include file "mpif.h".                                                :::
+#                                                                       :::
+#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+
+export which_MPI=openmpi                       # default, overwritten below
+
+parallel=0
+clean=1
+dprint=0
+
+export MY_CPP_FLAGS=
+
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    -j )
+      shift
+      parallel=1
+      test=`echo $1 | grep '^[0-9]\+$'`
+      if [ "$test" != "" ]; then
+        NCPUS="-j $1"
+        shift
+      else
+        NCPUS="-j"
+      fi
+      ;;
+
+    -p )
+      shift
+      clean=0
+      dprint=1
+      debug="print-$1"
+      shift
+      ;;
+
+    -noclean )
+      shift
+      clean=0
+      ;;
+
+    * )
+      echo ""
+      echo "$0 : Unknown option [ $1 ]"
+      echo ""
+      echo "Available Options:"
+      echo ""
+      echo "-j [N]      Compile in parallel using N CPUs"
+      echo "              omit argument for all avaliable CPUs"
+      echo ""
+      echo "-p macro    Prints any Makefile macro value"
+      echo "              For example:  build_roms.sh -p FFLAGS"
+      echo ""
+      echo "-noclean    Do not clean already compiled objects"
+      echo ""
+      exit 1
+      ;;
+  esac
+done
+
+# Set the CPP option defining the particular application. This will
+# determine the name of the ".h" header file with the application
+# CPP definitions.
+
+export   ROMS_APPLICATION=UPWELLING
+
+# Set a local environmental variable to define the path to the directories
+# where all this project's files are kept.
+
+export        MY_ROOT_DIR=${JARVIS_ROOT}
+export     MY_PROJECT_DIR=${MY_ROOT_DIR}/ROMS/bin
+
+# The path to the user's local current ROMS source code.
+#
+# If using svn locally, this would be the user's Working Copy Path (WCPATH).
+# Note that one advantage of maintaining your source code locally with svn
+# is that when working simultaneously on multiple machines (e.g. a local
+# workstation, a local cluster and a remote supercomputer) you can checkout
+# the latest release and always get an up-to-date customized source on each
+# machine. This script is designed to more easily allow for differing paths
+# to the code and inputs on differing machines.
+
+ export       MY_ROMS_SRC=${MY_ROOT_DIR}/roms-roms-4.0
+
+# Set path of the directory containing makefile configuration (*.mk) files.
+# The user has the option to specify a customized version of these files
+# in a different directory than the one distributed with the source code,
+# ${MY_ROMS_SRC}/Compilers. If this is the case, you need to keep these
+# configurations files up-to-date.
+
+ export         COMPILERS=${MY_ROMS_SRC}/Compilers
+#export         COMPILERS=${HOME}/Compilers/ROMS
+
+#--------------------------------------------------------------------------
+# Set tunable CPP options.
+#--------------------------------------------------------------------------
+#
+# Sometimes it is desirable to activate one or more CPP options to run
+# different variants of the same application without modifying its header
+# file. If this is the case, specify each options here using the -D syntax.
+# Notice also that you need to use shell's quoting syntax to enclose the
+# definition.  Both single or double quotes work. For example,
+#
+#export      MY_CPP_FLAGS="${MY_CPP_FLAGS} -DAVERAGES"
+#export      MY_CPP_FLAGS="${MY_CPP_FLAGS} -DDEBUGGING"
+#
+# can be used to write time-averaged fields. Notice that you can have as
+# many definitions as you want by appending values.
+
+#export      MY_CPP_FLAGS="${MY_CPP_FLAGS} -D"
+
+#--------------------------------------------------------------------------
+# Compiler options.
+#--------------------------------------------------------------------------
+#
+# Other user defined environmental variables. See the ROMS makefile for
+# details on other options the user might want to set here. Be sure to
+# leave the switches meant to be off set to an empty string or commented
+# out. Any string value (including off) will evaluate to TRUE in
+# conditional if-statements.
+
+ export           USE_MPI=on            # distributed-memory parallelism
+ export        USE_MPIF90=on            # compile with mpif90 script
+#export         which_MPI=intel         # compile with mpiifort library
+#export         which_MPI=mpich         # compile with MPICH library
+#export         which_MPI=mpich2        # compile with MPICH2 library
+#export         which_MPI=mvapich2      # compile with MVAPICH2 library
+ export         which_MPI=openmpi       # compile with OpenMPI library
+
+#export        USE_OpenMP=on            # shared-memory parallelism
+
+ export              FORT=flang
+#export              FORT=gfortran
+#export              FORT=pgi
+
+#export         USE_DEBUG=on            # use Fortran debugging flags
+ export         USE_LARGE=on            # activate 64-bit compilation
+
+# ROMS I/O choices and combinations. A more complete description of the
+# available options can be found in the wiki (https://myroms.org/wiki/IO).
+# Most users will want to enable at least USE_NETCDF4 because that will
+# instruct the ROMS build system to use nf-config to determine the
+# necessary libraries and paths to link into the ROMS executable.
+
+ export       USE_NETCDF4=on            # compile with NetCDF-4 library
+ export   USE_PARALLEL_IO=on            # Parallel I/O with NetCDF-4/HDF5
+#export           USE_PIO=on            # Parallel I/O with PIO library
+#export       USE_SCORPIO=on            # Parallel I/O with SCORPIO library
+
+# If any of the coupling component use the HDF5 Fortran API for primary
+# I/O, we need to compile the main driver with the HDF5 library.
+
+export          USE_HDF5=on            # compile with HDF5 library
+
+#--------------------------------------------------------------------------
+# If Earth System Model (ESM) coupling, set location of ESM component
+# libraries and modules. The strategy is to compile and link each ESM
+# component separately first and ROMS last since it is driving the
+# coupled system. Only the ESM components activated are considered and
+# the rest ignored.
+#--------------------------------------------------------------------------
+
+export        WRF_SRC_DIR=${HOME}/ocean/repository/WRF
+
+if [ -n "${USE_DEBUG:+1}" ]; then
+  export     CICE_LIB_DIR=${MY_PROJECT_DIR}/Build_ciceG
+  export   COAMPS_LIB_DIR=${MY_PROJECT_DIR}/Build_coampsG
+  export    REGCM_LIB_DIR=${MY_PROJECT_DIR}/Build_regcmG
+  export      WAM_LIB_DIR=${MY_PROJECT_DIR}/Build_wamG
+# export      WRF_LIB_DIR=${MY_PROJECT_DIR}/Build_wrfG
+  export      WRF_LIB_DIR=${WRF_SRC_DIR}
+else
+  export     CICE_LIB_DIR=${MY_PROJECT_DIR}/Build_cice
+  export   COAMPS_LIB_DIR=${MY_PROJECT_DIR}/Build_coamps
+  export    REGCM_LIB_DIR=${MY_PROJECT_DIR}/Build_regcm
+  export      WAM_LIB_DIR=${MY_PROJECT_DIR}/Build_wam
+  export      WRF_LIB_DIR=${MY_PROJECT_DIR}/Build_wrf
+# export      WRF_LIB_DIR=${WRF_SRC_DIR}
+fi
+
+#--------------------------------------------------------------------------
+# If applicable, use my specified library paths.
+#--------------------------------------------------------------------------
+
+#export USE_MY_LIBS=no            # use system default library paths
+ export USE_MY_LIBS=yes           # use my customized library paths
+
+MY_PATHS=${COMPILERS}/my_build_paths.sh
+
+if [ "${USE_MY_LIBS}" = "yes" ]; then
+  source ${MY_PATHS} ${MY_PATHS}
+fi
+
+#--------------------------------------------------------------------------
+# The rest of this script sets the path to the users header file and
+# analytical source files, if any. See the templates in User/Functionals.
+#--------------------------------------------------------------------------
+#
+# If applicable, use the MY_ANALYTICAL_DIR directory to place your
+# customized biology model header file (like fennel.h, nemuro.h, ecosim.h,
+# etc).
+
+ export     MY_HEADER_DIR=${MY_PROJECT_DIR}
+
+ export MY_ANALYTICAL_DIR=${MY_PROJECT_DIR}
+
+# Put the binary to execute in the following directory.
+
+ export            BINDIR=${MY_PROJECT_DIR}
+
+# Put the f90 files in a project specific Build directory to avoid conflict
+# with other projects.
+
+if [ -n "${USE_DEBUG:+1}" ]; then
+ export       SCRATCH_DIR=${MY_PROJECT_DIR}/Build_romsG
+else
+ export       SCRATCH_DIR=${MY_PROJECT_DIR}/Build_roms
+fi
+
+# Go to the users source directory to compile. The options set above will
+# pick up the application-specific code from the appropriate place.
+
+ cd ${MY_ROMS_SRC}
+
+# Stop if activating both MPI and OpenMP at the same time.
+
+if [ -n "${USE_MPI:+1}" ] && [ -n "${USE_OpenMP:+1}" ]; then
+  echo "You cannot activate USE_MPI and USE_OpenMP at the same time!"
+  exit 1
+fi
+
+#--------------------------------------------------------------------------
+# Compile.
+#--------------------------------------------------------------------------
+
+# Remove build directory.
+
+if [ $clean -eq 1 ]; then
+  make clean
+fi
+
+# Compile (the binary will go to BINDIR set above).
+
+if [ $dprint -eq 1 ]; then
+  make $debug
+else
+  if [ $parallel -eq 1 ]; then
+    make $NCPUS
+  else
+    make
+  fi
+fi
diff --git a/templates/roms/4.0/my_build_paths.sh b/templates/roms/4.0/my_build_paths.sh
new file mode 100755
index 0000000..6c26c83
--- /dev/null
+++ b/templates/roms/4.0/my_build_paths.sh
@@ -0,0 +1,375 @@
+#!/bin/bash
+#
+# svn $Id: my_build_paths.sh 1064 2021-05-10 19:55:56Z arango $
+#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+# Copyright (c) 2002-2021 The ROMS/TOMS Group                           :::
+#   Licensed under a MIT/X style license                                :::
+#   See License_ROMS.txt                                                :::
+#::::::::::::::::::::::::::::::::::::::::::::::::::::: Hernan G. Arango :::
+#                                                                       :::
+# ROMS Customized Compiling Libraries BASH Script:                      :::
+#                                                                       :::
+# This bash script sets the customized library paths needed by the      :::
+# build script when the enviromental variable USE_MY_LIBS has a 'yes'   :::
+# value.                                                                :::
+#                                                                       :::
+# For example, in build_roms.sh we have:                                :::
+#                                                                       :::
+#       if [ "${USE_MY_LIBS}" = "yes" ]; then                           :::
+#         source ${COMPILERS}/my_build_paths.sh                         :::
+#       fi                                                              :::
+#                                                                       :::
+#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+
+separator=`perl -e "print ':' x 100;"`
+
+echo ""
+echo "${separator}"
+echo "Using customized library paths from:  $1"
+echo "${separator}"
+echo ""
+
+#--------------------------------------------------------------------------
+# Add MPI library to compile.
+#--------------------------------------------------------------------------
+#
+# Recall also that the MPI library comes in several flavors:
+# MPICH, MPICH2, OpenMPI, etc.
+
+# There are several MPI libraries available (MPICH, MPICH2, OpenMPI,
+# etc.). Here, we set the desired  "mpif90" script to use during
+# compilation. This only works if the make configuration file (say,
+# Linux-ifort.mk) in the "Compilers" directory has the following
+# definition for FC (Fortran Compiler) in the USE_MPI section:
+#
+#              FC := mpif90
+#
+# that is, "mpif90" defined without any path. Notice that the path
+# where the MPI library is installed is computer dependent. Recall
+# that you still need to use the appropriate "mpirun" to execute.
+
+export               MPI_ROOT=""
+
+if [ -n "${USE_MPIF90:+1}" ]; then
+  case "$FORT" in
+    ifort )
+      if [ "${which_MPI}" = "mpich" ]; then
+        export       MPI_ROOT=/opt/intelsoft/mpich
+      elif [ "${which_MPI}" = "mpich2" ]; then
+        export       MPI_ROOT=/opt/intelsoft/mpich2
+      elif [ "${which_MPI}" = "openmpi" ]; then
+        export       MPI_ROOT=/opt/intelsoft/openmpi
+      elif [ "${which_MPI}" = "mvapich2" ]; then
+        export       MPI_ROOT=/opt/intelsoft/mvapich2
+      fi
+      export             PATH=${MPI_ROOT}/bin:$PATH
+      export       MPI_INCDIR=${MPI_ROOT}/include
+      ;;
+
+    pgi )
+      if [ "${which_MPI}" = "mpich" ]; then
+        export       MPI_ROOT=/opt/pgisoft/mpich
+      elif [ "${which_MPI}" = "mpich2" ]; then
+        export       MPI_ROOT=/opt/pgisoft/mpich2
+      elif [ "${which_MPI}" = "openmpi" ]; then
+        export       MPI_ROOT=/opt/pgisoft/openmpi
+      elif [ "${which_MPI}" = "mvapich2" ]; then
+        export       MPI_ROOT=/opt/pgisoft/mvapich2
+      fi
+      export             PATH=${MPI_ROOT}/bin:$PATH
+      export       MPI_INCDIR=${MPI_ROOT}/include
+      ;;
+
+    flang )
+      if [ "${which_MPI}" = "mpich2" ]; then
+        export       MPI_ROOT=/opt/gfortransoft/mpich2
+      elif [ "${which_MPI}" = "openmpi" ]; then
+	export       MPI_ROOT=${HMPI_PATH}
+      elif [ "${which_MPI}" = "mvapich2" ]; then
+        export       MPI_ROOT=/opt/gfortransoft/mvapich2
+      fi
+      export             PATH=${MPI_ROOT}/bin:$PATH
+      export       MPI_INCDIR=${MPI_ROOT}/include
+      ;;
+
+  esac
+fi
+
+#--------------------------------------------------------------------------
+# Set libraries to compile and link.
+#--------------------------------------------------------------------------
+#
+# The path of the libraries required by ROMS can be set here using
+# environmental variables which take precedence to the values
+# specified in the make macro definitions file (Compilers/*.mk).
+# For most applications, only the location of the NetCDF library
+# is needed during compilation.
+#
+# Notice that when the USE_NETCDF4 macro is activated, we need the
+# serial or parallel version of the NetCDF-4/HDF5 library. The
+# configuration script NF_CONFIG (available since NetCDF 4.0.1)
+# is used to set up all the required libraries according to the
+# installed options (openDAP, netCDF4/HDF5 file format). The
+# parallel library uses the MPI-I/O layer (usually available
+# in MPICH2 and OpenMPI) requiring compiling with the selected
+# MPI library.
+#
+# In ROMS distributed-memory applications, you may use either the
+# serial or parallel version of the NetCDF-4/HDF5 library. The
+# parallel version is required when parallel I/O is activated
+# (ROMS cpp option PARALLEL_IO and HDF5).
+#
+# However, in serial or shared-memory ROMS applications, we need
+# to use the serial version of the NetCDF-4/HDF5 to avoid conflicts
+# with the compiler. We cannot activate MPI constructs in serial
+# or shared-memory ROMS code. Hybrid parallelism is not possible.
+
+export               MPI_SOFT=""
+
+case "$FORT" in
+  ifort )
+    export      ESMF_COMPILER=intelgcc
+#   export      ESMF_COMPILER=intel
+    if [ -n "${USE_DEBUG:+1}" ]; then
+      export        ESMF_BOPT=g
+    else
+      export        ESMF_BOPT=O
+    fi
+    export           ESMF_ABI=64
+    export          ESMF_COMM=${which_MPI}
+    export          ESMF_SITE=default
+
+    export      ARPACK_LIBDIR=/opt/intelsoft/serial/ARPACK
+    if [ -n "${USE_MPI:+1}" ]; then
+      if [ "${which_MPI}" = "mpich" ]; then
+        export       MPI_SOFT=/opt/intelsoft/mpich
+      elif [ "${which_MPI}" = "mpich2" ]; then
+        export       MPI_SOFT=/opt/intelsoft/mpich2
+      elif [ "${which_MPI}" = "openmpi" ]; then
+        export       MPI_SOFT=/opt/intelsoft/openmpi
+      elif [ "${which_MPI}" = "mvapich2" ]; then
+        export       MPI_SOFT=/opt/intelsoft/mvapich2
+      fi
+      export       MCT_INCDIR=${MPI_SOFT}/mct/include
+      export       MCT_LIBDIR=${MPI_SOFT}/mct/lib
+      export   PARPACK_LIBDIR=${MPI_SOFT}/PARPACK
+    fi
+
+    if [ ! -n "${SINGULARITY_COMMAND:+1}" ]; then
+      if [ -n "${USE_NETCDF4:+1}" ]; then
+        if [ -n "${USE_PARALLEL_IO:+1}" ] && [ -n "${USE_MPI:+1}" ]; then
+          export       ESMF_DIR=${MPI_SOFT}/esmf_nc4
+          export         NETCDF=${MPI_SOFT}/netcdf4
+          export      NF_CONFIG=${NETCDF}/bin/nf-config
+          export  NETCDF_INCDIR=${NETCDF}/include
+          export        NETCDF4=1
+        else
+          export       ESMF_DIR=${MPI_SOFT}/esmf_nc4
+          export         NETCDF=/opt/intelsoft/serial/netcdf4
+          export      NF_CONFIG=${NETCDF}/bin/nf-config
+          export  NETCDF_INCDIR=${NETCDF}/include
+          export        NETCDF4=1
+        fi
+      else
+        export         ESMF_DIR=${MPI_SOFT}/esmf_nc3
+        export           NETCDF=/opt/intelsoft/serial/netcdf3
+        export    NETCDF_INCDIR=${NETCDF}/include
+        export    NETCDF_LIBDIR=${NETCDF}/lib
+        export   NETCDF_classic=1
+      fi
+    fi
+
+    if [ -n "${USE_PIO:+1}" ]; then
+      export              PIO=${MPI_SOFT}/pio
+      export       PIO_LIBDIR=${PIO}/lib
+      export       PIO_INCDIR=${PIO}/include
+      export          PNETCDF=${MPI_SOFT}/pnetcdf
+      export   PNETCDF_LIBDIR=${PNETCDF}/lib
+      export   PNETCDF_INCDIR=${PNETCDF}/include
+    fi
+
+    if [ -n "${USE_SCORPIO:+1}" ]; then
+      export          SCORPIO=${MPI_SOFT}/scorpio
+      export       PIO_LIBDIR=${SCORPIO}/lib
+      export       PIO_INCDIR=${SCORPIO}/include
+      export          PNETCDF=${MPI_SOFT}/pnetcdf
+      export   PNETCDF_LIBDIR=${PNETCDF}/lib
+      export   PNETCDF_INCDIR=${PNETCDF}/include
+    fi
+
+    if [ -n "${USE_HDF5:+1}" ]; then
+      if [ -n "${USE_PARALLEL_IO:+1}" ] && [ -n "${USE_MPI:+1}" ]; then
+        export           HDF5=${MPI_SOFT}/hdf5
+        export    HDF5_LIBDIR=${HDF5}/lib
+        export    HDF5_INCDIR=${HDF5}/include
+      else
+        export           HDF5=/opt/intelsoft/serial/hdf5
+        export    HDF5_LIBDIR=${HDF5}/lib
+        export    HDF5_INCDIR=${HDF5}/include
+      fi
+    fi
+    ;;
+
+  pgi )
+    export      ESMF_COMPILER=pgi
+    if [ -n "${USE_DEBUG:+1}" ]; then
+      export        ESMF_BOPT=g
+    else
+      export        ESMF_BOPT=O
+    fi
+    export           ESMF_ABI=64
+    export          ESMF_COMM=${which_MPI}
+    export          ESMF_SITE=default
+
+    export      ARPACK_LIBDIR=/opt/pgisoft/serial/ARPACK
+    if [ -n "${USE_MPI:+1}" ]; then
+      if [ "${which_MPI}" = "mpich" ]; then
+        export       MPI_SOFT=/opt/pgisoft/mpich
+      elif [ "${which_MPI}" = "mpich2" ]; then
+        export       MPI_SOFT=/opt/pgisoft/mpich2
+      elif [ "${which_MPI}" = "openmpi" ]; then
+        export       MPI_SOFT=/opt/pgisoft/openmpi
+      elif [ "${which_MPI}" = "mvapich2" ]; then
+        export       MPI_SOFT=/opt/pgisoft/mvapich2
+      fi
+      export       MCT_INCDIR=${MPI_SOFT}/mct/include
+      export       MCT_LIBDIR=${MPI_SOFT}/mct/lib
+      export   PARPACK_LIBDIR=${MPI_SOFT}/PARPACK
+    fi
+
+    if [ ! -n "${SINGULARITY_COMMAND:+1}" ]; then
+      if [ -n "${USE_NETCDF4:+1}" ]; then
+        if [ -n "${USE_PARALLEL_IO:+1}" ] && [ -n "${USE_MPI:+1}" ]; then
+          export       ESMF_DIR=${MPI_SOFT}/esmf_nc4
+          export         NETCDF=${MPI_SOFT}/netcdf4
+          export      NF_CONFIG=${NETCDF}/bin/nf-config
+          export  NETCDF_INCDIR=${NETCDF}/include
+          export        NETCDF4=1
+        else
+          export       ESMF_DIR=${MPI_SOFT}/esmf_nc4
+          export         NETCDF=/opt/pgisoft/serial/netcdf4
+          export      NF_CONFIG=${NETCDF}/bin/nf-config
+          export  NETCDF_INCDIR=${NETCDF}/include
+          export        NETCDF4=1
+        fi
+      else
+        export         ESMF_DIR=${MPI_SOFT}/esmf_nc3
+        export           NETCDF=/opt/pgisoft/serial/netcdf3
+        export    NETCDF_INCDIR=${NETCDF}/include
+        export    NETCDF_LIBDIR=${NETCDF}/lib
+        export   NETCDF_classic=1
+      fi
+    fi
+
+    if [ -n "${USE_PIO:+1}" ]; then
+      export              PIO=${MPI_SOFT}/pio
+      export       PIO_LIBDIR=${PIO}/lib
+      export       PIO_INCDIR=${PIO}/include
+      export          PNETCDF=${MPI_SOFT}/pnetcdf
+      export   PNETCDF_LIBDIR=${PNETCDF}/lib
+      export   PNETCDF_INCDIR=${PNETCDF}/include
+    fi
+
+    if [ -n "${USE_SCORPIO:+1}" ]; then
+      export          SCORPIO=${MPI_SOFT}/scorpio
+      export       PIO_LIBDIR=${SCORPIO}/lib
+      export       PIO_INCDIR=${SCORPIO}/include
+      export          PNETCDF=${MPI_SOFT}/pnetcdf
+      export   PNETCDF_LIBDIR=${PNETCDF}/lib
+      export   PNETCDF_INCDIR=${PNETCDF}/include
+    fi
+
+    if [ -n "${USE_HDF5:+1}" ]; then
+      if [ -n "${USE_PARALLEL_IO:+1}" ] && [ -n "${USE_MPI:+1}" ]; then
+        export           HDF5=${MPI_SOFT}/hdf5
+        export    HDF5_LIBDIR=${HDF5}/lib
+        export    HDF5_INCDIR=${HDF5}/include
+      else
+        export           HDF5=/opt/pgisoft/serial/hdf5
+        export    HDF5_LIBDIR=${HDF5}/lib
+        export    HDF5_INCDIR=${HDF5}/include
+      fi
+    fi
+    ;;
+
+  flang )
+    export      ESMF_COMPILER=flang
+    if [ -n "${USE_DEBUG:+1}" ]; then
+      export        ESMF_BOPT=g
+    else
+      export        ESMF_BOPT=O
+    fi
+    export           ESMF_ABI=64
+    export          ESMF_COMM=${which_MPI}
+    export          ESMF_SITE=default
+
+    export      ARPACK_LIBDIR=/opt/gfortransoft/serial/ARPACK
+    if [ -n "${USE_MPI:+1}" ]; then
+      if [ "${which_MPI}" = "mpich2" ]; then
+        export       MPI_SOFT=/opt/gfortransoft/mpich2
+      elif [ "${which_MPI}" = "openmpi" ]; then
+        export       MPI_SOFT=${HMPI_PATH}
+      elif [ "${which_MPI}" = "mvapich2" ]; then
+        export       MPI_SOFT=/opt/gfortransoft/mvapich2
+      fi
+      export       MCT_INCDIR=${MPI_SOFT}/mct/include
+      export       MCT_LIBDIR=${MPI_SOFT}/mct/lib
+      export   PARPACK_LIBDIR=${MPI_SOFT}/PARPACK
+    fi
+
+    if [ ! -n "${SINGULARITY_COMMAND:+1}" ]; then
+      if [ -n "${USE_NETCDF4:+1}" ]; then
+        if [ -n "${USE_PARALLEL_IO:+1}" ] && [ -n "${USE_MPI:+1}" ]; then
+          export       ESMF_DIR=${MPI_SOFT}/esmf_nc4
+          export         NETCDF=${NETCDF_CLANG_PATH}
+          export      NF_CONFIG=${NETCDF}/bin/nf-config
+          export  NETCDF_INCDIR=${NETCDF}/include
+          export        NETCDF4=1
+        else
+          export       ESMF_DIR=${MPI_SOFT}/esmf_nc4
+          export         NETCDF=${NETCDF_CLANG_PATH}
+          export      NF_CONFIG=${NETCDF}/bin/nf-config
+          export  NETCDF_INCDIR=${NETCDF}/include
+          export        NETCDF4=1
+        fi
+      else
+        export         ESMF_DIR=${MPI_SOFT}/esmf_nc3
+        export           NETCDF=/opt/gfortransoft/serial/netcdf3
+        export    NETCDF_INCDIR=${NETCDF}/include
+        export    NETCDF_LIBDIR=${NETCDF}/lib
+        export   NETCDF_classic=1
+      fi
+    fi
+
+    if [ -n "${USE_PIO:+1}" ]; then
+      export              PIO=${PIO_PATH}
+      export       PIO_LIBDIR=${PIO}/lib
+      export       PIO_INCDIR=${PIO}/include
+      export          PNETCDF=${PNETCDF_PATH}
+      export   PNETCDF_LIBDIR=${PNETCDF}/lib
+      export   PNETCDF_INCDIR=${PNETCDF}/include
+    fi
+
+    if [ -n "${USE_SCORPIO:+1}" ]; then
+      export          SCORPIO=${MPI_SOFT}/scorpio
+      export       PIO_LIBDIR=${SCORPIO}/lib
+      export       PIO_INCDIR=${SCORPIO}/include
+      export          PNETCDF=${MPI_SOFT}/pnetcdf
+      export   PNETCDF_LIBDIR=${PNETCDF}/lib
+      export   PNETCDF_INCDIR=${PNETCDF}/include
+    fi
+
+    if [ -n "${USE_HDF5:+1}" ]; then
+      if [ -n "${USE_PARALLEL_IO:+1}" ] && [ -n "${USE_MPI:+1}" ]; then
+        export           HDF5=${HDF5_CLANG_PATH}
+        export    HDF5_LIBDIR=${HDF5}/lib
+        export    HDF5_INCDIR=${HDF5}/include
+      else
+        export           HDF5=${HDF5_CLANG_PATH}
+        export    HDF5_LIBDIR=${HDF5}/lib
+        export    HDF5_INCDIR=${HDF5}/include
+      fi
+    fi
+    ;;
+
+esac
diff --git a/templates/roms/4.0/roms.singularity.arm.bisheng-sve.cpu.config b/templates/roms/4.0/roms.singularity.arm.bisheng-sve.cpu.config
new file mode 100644
index 0000000..3324394
--- /dev/null
+++ b/templates/roms/4.0/roms.singularity.arm.bisheng-sve.cpu.config
@@ -0,0 +1,81 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+ROMS/4.0 $JARVIS_PROXY/myroms/roms/archive/refs/tags/roms-4.0.tar.gz
+
+[DEPENDENCY]
+set -x
+set -e
+#yum install perl-XML-LibXML
+./jarvis -install package/bisheng/3.2.0-mini com
+module purge
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=`which clang`
+export CXX=`which clang++`
+export FC=`which flang`
+./jarvis -install package/hmpi/2.3.0 bisheng
+module use software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+export CC=mpicc CXX=mpicxx FC=mpifort F77=mpifort
+./jarvis -install package/hdf5/1.12.0/clang bisheng+mpi
+module load hdf5-clang/1.12.0
+./jarvis -install pnetcdf/1.12.3 bisheng+mpi
+module load pnetcdf/1.12.3
+./jarvis -install netcdf/4.7.4/clang bisheng+mpi
+module load netcdf-clang/4.7.4
+
+[ENV]
+module purge
+module use software/modulefiles
+module load bisheng/3.2.0-mini
+module use software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load hdf5-clang/1.12.0
+module load pnetcdf/1.12.3
+module load netcdf-clang/4.7.4
+
+export NETCDF_PATH=${NETCDF_CLANG_PATH}
+export CC=mpicc CXX=mpicxx FC=mpifort F77=mpifort
+export PATH=$JARVIS_ROOT/ROMS/bin:$PATH
+
+[APP]
+app_name = ROMS
+build_dir = $JARVIS_ROOT/ROMS/bin 
+binary_dir = $JARVIS_ROOT/ROMS/bin
+case_dir = $JARVIS_ROOT/ROMS/bin 
+
+[BUILD]
+set -e
+set -x
+
+#cp /mnt/src_roms_nesting.tar.gz $JARVIS_ROOT/
+
+cd ${JARVIS_ROOT}
+tar -xf $JARVIS_DOWNLOAD/roms-4.0.tar.gz 
+
+if [ ! -d ROMS ] ; then
+ mkdir -p ROMS/bin 
+fi
+
+cd ROMS/bin
+cp $JARVIS_ROOT/templates/roms/4.0/build_roms.sh ./
+cp $JARVIS_ROOT/templates/roms/4.0/my_build_paths.sh $JARVIS_ROOT/roms-roms-4.0/Compilers
+cp $JARVIS_ROOT/templates/roms/4.0/Linux-flang.mk $JARVIS_ROOT/roms-roms-4.0/Compilers
+cp $JARVIS_ROOT/roms-roms-4.0/ROMS/External/roms_upwelling.in .
+cp $JARVIS_ROOT/roms-roms-4.0/ROMS/External/varinfo.dat .
+cp $JARVIS_ROOT/roms-roms-4.0/ROMS/Include/upwelling.h .
+sed -i "76s%ROMS/External/varinfo.dat%./varinfo.dat%" roms_upwelling.in
+
+sed -i "s/-mcpu=tsv110/-mcpu=linxicore9100 -mllvm -force-customized-pipeline=true/g" $JARVIS_ROOT/roms-roms-4.0/Compilers/Linux-flang.mk
+chmod +x build_roms.sh
+./build_roms.sh -j 8
+
+[RUN]
+run = chmod +x submit_roms.sh && dsub -s submit_roms.sh
+binary = romsM
+nodes = 1
+
+[BATCH]
+#! /bin/bash
diff --git a/templates/roms/4.0/roms.singularity.arm.bisheng.cpu.config b/templates/roms/4.0/roms.singularity.arm.bisheng.cpu.config
new file mode 100644
index 0000000..ea36bf5
--- /dev/null
+++ b/templates/roms/4.0/roms.singularity.arm.bisheng.cpu.config
@@ -0,0 +1,79 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+ROMS/4.0 $JARVIS_PROXY/myroms/roms/archive/refs/tags/roms-4.0.tar.gz
+
+[DEPENDENCY]
+set -x
+set -e
+#yum install perl-XML-LibXML
+./jarvis -install package/bisheng/3.2.0-mini com
+module purge
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=`which clang`
+export CXX=`which clang++`
+export FC=`which flang`
+./jarvis -install package/hmpi/2.3.0 bisheng
+module use software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+export CC=mpicc CXX=mpicxx FC=mpifort F77=mpifort
+./jarvis -install package/hdf5/1.12.0/clang bisheng+mpi
+module load hdf5-clang/1.12.0
+./jarvis -install pnetcdf/1.12.3 bisheng+mpi
+module load pnetcdf/1.12.3
+./jarvis -install netcdf/4.7.4/clang bisheng+mpi
+module load netcdf-clang/4.7.4
+
+[ENV]
+module purge
+module use software/modulefiles
+module load bisheng/3.2.0-mini
+module use software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load hdf5-clang/1.12.0
+module load pnetcdf/1.12.3
+module load netcdf-clang/4.7.4
+
+export NETCDF_PATH=${NETCDF_CLANG_PATH}
+export CC=mpicc CXX=mpicxx FC=mpifort F77=mpifort
+export PATH=$JARVIS_ROOT/ROMS/bin:$PATH
+
+[APP]
+app_name = ROMS
+build_dir = $JARVIS_ROOT/ROMS/bin 
+binary_dir = $JARVIS_ROOT/ROMS/bin
+case_dir = $JARVIS_ROOT/ROMS/bin 
+
+[BUILD]
+set -e
+set -x
+
+#cp /mnt/src_roms_nesting.tar.gz $JARVIS_ROOT/
+
+cd ${JARVIS_ROOT}
+tar -xf $JARVIS_DOWNLOAD/roms-4.0.tar.gz
+
+if [ ! -d ROMS ] ; then
+ mkdir -p ROMS/bin
+fi
+cd ROMS/bin
+cp $JARVIS_ROOT/templates/roms/4.0/build_roms.sh ./
+cp $JARVIS_ROOT/templates/roms/4.0/my_build_paths.sh $JARVIS_ROOT/roms-roms-4.0/Compilers
+cp $JARVIS_ROOT/templates/roms/4.0/Linux-flang.mk $JARVIS_ROOT/roms-roms-4.0/Compilers
+cp $JARVIS_ROOT/roms-roms-4.0/ROMS/External/roms_upwelling.in .
+cp $JARVIS_ROOT/roms-roms-4.0/ROMS/External/varinfo.dat .
+cp $JARVIS_ROOT/roms-roms-4.0/ROMS/Include/upwelling.h .
+sed -i "76s%ROMS/External/varinfo.dat%./varinfo.dat%" roms_upwelling.in
+
+chmod +x build_roms.sh
+./build_roms.sh -j 8
+
+[RUN]
+run = chmod +x submit_roms.sh && dsub -s submit_roms.sh
+binary = romsM
+nodes = 1
+
+[BATCH]
+#! /bin/bash
-- 
Gitee


From c3b784ead3ea44758e4dc6de762d0430ceec0dd1 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 19 Mar 2024 11:17:52 +0800
Subject: [PATCH 16/19] add package/hmpi/1.3.0/install.sh

---
 package/hmpi/1.3.0/install.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/package/hmpi/1.3.0/install.sh b/package/hmpi/1.3.0/install.sh
index b78fdf3..c0c087f 100755
--- a/package/hmpi/1.3.0/install.sh
+++ b/package/hmpi/1.3.0/install.sh
@@ -2,14 +2,14 @@
 set -x
 set -e
 hmpi_version='1.3.0'
-. ${DOWNLOAD_TOOL} -u $JARVIS_PROXY/openucx/ucx/archive/refs/tags/v1.10.1.tar.gz -f hucx-${hmpi_version}-huawei.tar.gz
-. ${DOWNLOAD_TOOL} -u $JARVIS_PROXY/chen-shaoheng/HMPI-1.3.0/releases/download/xucg-v1.3.0/xucg-v1.3.0-huawei.tar.gz -f xucg-${hmpi_version}-huawei.tar.gz
-. ${DOWNLOAD_TOOL} -u $JARVIS_PROXY/chen-shaoheng/HMPI-1.3.0/releases/download/hmpi-v1.3.0/hmpi-v1.3.0-huawei.tar.gz -f hmpi-${hmpi_version}-huawei.tar.gz
+. ${DOWNLOAD_TOOL} -u https://gitee.com/kunpengcompute/hucx/repository/archive/v1.3.0-huawei.tar.gz -f hucx-${hmpi_version}-huawei.tar.gz
+. ${DOWNLOAD_TOOL} -u https://gitee.com/kunpengcompute/xucg/repository/archive/v1.3.0-huawei.tar.gz -f xucg-${hmpi_version}-huawei.tar.gz
+. ${DOWNLOAD_TOOL} -u https://gitee.com/kunpengcompute/hmpi/repository/archive/v1.3.0-huawei.tar.gz -f hmpi-${hmpi_version}-huawei.tar.gz
 cd ${JARVIS_TMP}
 . $CHECK_ROOT && yum install -y perl-Data-Dumper autoconf automake libtool binutils flex
-rm ucx-1.10.1 -rf
-tar xf ${JARVIS_DOWNLOAD}/hucx-1.3.0-huawei.tar.gz
-cd ucx-1.10.1
+rm hucx-v${hmpi_version}-huawei -rf
+tar xf ${JARVIS_DOWNLOAD}/hucx-${hmpi_version}-huawei.tar.gz
+cd hucx-v${hmpi_version}-huawei
 ./autogen.sh
 ./contrib/configure-opt --prefix=$1/hucx --disable-numa --enable-mt CFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion" CXXFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion"
 make -j
@@ -21,9 +21,9 @@ export CPLUS_INCLUDE_PATH=$1/hucx/include:$CPLUS_INCLUDE_PATH
 
 export CFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion" CXXFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion" FFLAGS="-Wno-error=int-conversion"
 
-rm xucg-v1.3.0-huawei/ -rf
-tar xf ${JARVIS_DOWNLOAD}/xucg-1.3.0-huawei.tar.gz
-cd xucg-v1.3.0-huawei/
+rm xucg-v${hmpi_version}-huawei/ -rf
+tar xf ${JARVIS_DOWNLOAD}/xucg-${hmpi_version}-huawei.tar.gz
+cd xucg-v${hmpi_version}-huawei/
 mkdir build && cd build
 cmake .. -DCMAKE_INSTALL_PREFIX=$1/xucg -DCMAKE_BUILD_TYPE=Release -DUCG_BUILD_WITH_UCX=$1/hucx -DUCG_ENABLE_MT=ON -DUCG_BUILD_TESTS=OFF -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_Fortran_COMPILER=$FC
 make -j
@@ -33,9 +33,9 @@ export LD_LIBRARY_PATH=$1/xucg/lib:$LD_LIBRARY_PATH
 export C_INCLUDE_PATH=$1/xucg/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=$1/xucg/include:$CPLUS_INCLUDE_PATH
 
-rm hmpi-v1.3.0-huawei/ -rf
-tar xf ${JARVIS_DOWNLOAD}/hmpi-1.3.0-huawei.tar.gz
-cd hmpi-v1.3.0-huawei/
+rm hmpi-v${hmpi_version}-huawei/ -rf
+tar xf ${JARVIS_DOWNLOAD}/hmpi-${hmpi_version}-huawei.tar.gz
+cd hmpi-v${hmpi_version}-huawei/
 ./autogen.pl
 ./configure --prefix=$1 --with-platform=contrib/platform/mellanox/optimized --enable-mpi1-compatibility --with-ucx=$1/hucx --with-ucg=$1/xucg
 make -j
-- 
Gitee


From fd9212d639c6fddb30ce9596ef5a23af2abdcc21 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 25 Mar 2024 11:20:22 +0800
Subject: [PATCH 17/19] fix qe/7.3.config

---
 templates/qe/7.3/data.qe.arm-sve.cpu.config | 1 +
 1 file changed, 1 insertion(+)

diff --git a/templates/qe/7.3/data.qe.arm-sve.cpu.config b/templates/qe/7.3/data.qe.arm-sve.cpu.config
index c9de3a5..d810a68 100644
--- a/templates/qe/7.3/data.qe.arm-sve.cpu.config
+++ b/templates/qe/7.3/data.qe.arm-sve.cpu.config
@@ -13,6 +13,7 @@ export CC=clang CXX=clang++ FC=flang
 ./jarvis -install hmpi/2.3.0 bisheng
 module use ./software/moduledeps/bisheng3.2.0
 module load hmpi/2.3.0
+./jarvis -install kml/2.2.0/bisheng bisheng
 module load kml-bisheng/2.2.0
 
 
-- 
Gitee


From 7293149616cfd0a229a728684b6f28f7f949483c Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Mon, 25 Mar 2024 17:37:25 +0800
Subject: [PATCH 18/19] add qe/7.3.config

---
 templates/qe/7.3/data.qe.arm-920.cpu.config | 60 +++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 templates/qe/7.3/data.qe.arm-920.cpu.config

diff --git a/templates/qe/7.3/data.qe.arm-920.cpu.config b/templates/qe/7.3/data.qe.arm-920.cpu.config
new file mode 100644
index 0000000..6bfe2e1
--- /dev/null
+++ b/templates/qe/7.3/data.qe.arm-920.cpu.config
@@ -0,0 +1,60 @@
+[SERVER]
+11.11.11.11
+
+[DOWNLOAD]
+qe/7.3 https://gitlab.com/QEF/q-e/-/archive/qe-7.3/q-e-qe-7.3.tar
+
+[DEPENDENCY]
+module purge 
+./jarvis -install bisheng/3.2.0-mini com
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+export CC=clang CXX=clang++ FC=flang
+./jarvis -install hmpi/2.3.0 bisheng
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+./jarvis -install kml/1.7.0/bisheng bisheng
+module load kml-bisheng/1.7.0
+
+
+[ENV]
+module purge
+module use ./software/modulefiles
+module load bisheng/3.2.0-mini
+module use ./software/moduledeps/bisheng3.2.0
+module load hmpi/2.3.0
+module load kml-bisheng/2.2.0
+
+export BLAS_LIBS="-L${JARVIS_ROOT}/software/libs/bisheng3.2.0/kml-bisheng/1.7.0/lib/kblas/omp -lkblas"
+export LAPACK_LIBS="-L${JARVIS_ROOT}/software/libs/bisheng3.2.0/kml-bisheng/1.7.0/lib -lklapack_full"
+
+cd /hpcrunner
+CUR_PATH=$(pwd)
+export JARVIS_ROOT=${CUR_PATH}
+export JARVIS_TMP=${CUR_PATH}/tmp
+export PATH=${JARVIS_ROOT}/q-e-qe-7.3-kml/bin:$PATH
+
+
+[APP]
+app_name = QE
+build_dir = ${JARVIS_TMP}/q-e-qe-7.3
+binary_dir = ${JARVIS_ROOT}/q-e-qe-7.3-kml/bin
+case_dir =
+
+[BUILD]
+cd ${JARVIS_TMP}
+tar -xf ${JARVIS_DOWNLOAD}/q-e-qe-7.3.tar
+cd q-e-qe-7.3
+
+./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3 -mcpu=native" CFLAGS="-O3 -mcpu=native" --with-scalapack=no --enable-openmp --prefix=$JARVIS_ROOT/q-e-qe-7.3-kml
+sed -i 's/gfortran/flang/g' make.inc
+make -j pwall
+make install
+
+[CLEAN]
+make clean
+
+[RUN]
+run =
+binary =
+nodes = 1
-- 
Gitee


From 9f32c9345daf31c4b30929e4a6227e2e56a460e3 Mon Sep 17 00:00:00 2001
From: chenshaoheng <chenshaoheng3@h-partners.com>
Date: Tue, 26 Mar 2024 11:42:03 +0800
Subject: [PATCH 19/19] add templates/CP2K/8.2/

---
 templates/CP2K/8.2/Linux-arm-clang-sve.psmp     | 13 ++++++++-----
 templates/CP2K/8.2/data.CP2K.arm-new.cpu.config | 16 ++++++++--------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/templates/CP2K/8.2/Linux-arm-clang-sve.psmp b/templates/CP2K/8.2/Linux-arm-clang-sve.psmp
index e1b8650..260d1fc 100644
--- a/templates/CP2K/8.2/Linux-arm-clang-sve.psmp
+++ b/templates/CP2K/8.2/Linux-arm-clang-sve.psmp
@@ -11,18 +11,20 @@ GNU_PATH        = path2
 
 include       $(MPI_PATH)/plumed/2.6.2/lib/plumed/src/lib/Plumed.inc.static
 
-ELPA_VER    = 2019.05.001
+ELPA_VER    = 2020.11.001
 ELPA_INC    = $(MPI_PATH)/elpa/$(ELPA_VER)/include/elpa_openmp-$(ELPA_VER)
 ELPA_LIB    = $(MPI_PATH)/elpa/$(ELPA_VER)/lib
 
+LIBVORI     = $(GNU_PATH)/libvori/21.04.12/lib
+
 FFTW_INC    = $(MPI_PATH)/fftw/3.3.8/include
 FFTW_LIB    = $(MPI_PATH)/fftw/3.3.8/lib
 
 LIBINT_INC  = $(MPI_PATH)/libint/2.6.0/include
 LIBINT_LIB  = $(MPI_PATH)/libint/2.6.0/lib
 
-LIBXC_INC   = $(GNU_PATH)/libxc/4.3.4/include
-LIBXC_LIB   = $(GNU_PATH)/libxc/4.3.4/lib
+LIBXC_INC   = $(GNU_PATH)/libxc/5.1.4/include
+LIBXC_LIB   = $(GNU_PATH)/libxc/5.1.4/lib
 
 SPGLIB_INC  = $(GNU_PATH)/spglib/1.16.0/include
 SPGLIB_LIB  = $(GNU_PATH)/spglib/1.16.0/lib64
@@ -30,7 +32,7 @@ SPGLIB_LIB  = $(GNU_PATH)/spglib/1.16.0/lib64
 #LIBXSMM_INC = $(GNU_PATH)/libxsmm/1.14/include
 #LIBXSMM_LIB = $(GNU_PATH)/libxsmm/1.14/lib
 
-CFLAGS      = -O2 -g -mtune=native -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true
+CFLAGS      = -O2 -g -mtune=native -fopenmp -Mbackslash -Hx,2,0x400000 -mcpu=linxicore9100 -mllvm -force-customized-pipeline=true
 
 DFLAGS      = -D__FFTW3 -D__LIBINT -D__LIBXC -D__ELPA  #-D__LIBXSMM -D__ELPA
 DFLAGS     += -D__MPI_VERSION=3 -D__PLUMED2 -D__SPGLIB
@@ -46,6 +48,7 @@ FCFLAGS    += -I$(FFTW_INC) -I$(LIBINT_INC) -I$(LIBXC_INC) #-I$(LIBXSMM_INC)
 LDFLAGS     = $(FCFLAGS) -static-libgfortran
 
 LIBS        = $(PLUMED_DEPENDENCIES) -L$(GNU_PATH)/gsl/2.6/lib/ -lgsl -lgslcblas -lz
+LIBS       += $(LIBVORI)/libvori.a
 LIBS       += $(ELPA_LIB)/libelpa_openmp.a
 LIBS       += $(LIBXC_LIB)/libxcf03.a
 LIBS       += $(LIBXC_LIB)/libxc.a
@@ -53,7 +56,7 @@ LIBS       += $(LIBINT_LIB)/libint2.a
 LIBS       += $(SPGLIB_LIB)/libsymspg.a
 LIBS       += $(FFTW_LIB)/libfftw3.a
 LIBS       += $(FFTW_LIB)/libfftw3_threads.a
-LIBS       += $(MPI_PATH)/scalapack-lapack-bisheng/2.1.0/lib/libscalapack.a
+LIBS       += $(MPI_PATH)/scalapack-clang/2.1.0/lib/libscalapack.a
 LIBS       += $(GNU_PATH)/lapack-bisheng/3.8.0/lib/liblapack.a
 LIBS       += $(GNU_PATH)/lapack-bisheng/3.8.0/lib/librefblas.a
 LIBS       += -ldl -lpthread -lstdc++
diff --git a/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config b/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config
index 8eaa92f..7a8a07b 100644
--- a/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config
+++ b/templates/CP2K/8.2/data.CP2K.arm-new.cpu.config
@@ -52,12 +52,12 @@ module load bisheng/3.2.0-mini
 module load boost/1.72.0  gmp/6.2.0  gsl/2.6  hmpi/2.3.0  lapack-bisheng/3.8.0  libvori/21.04.12  libxc/5.1.4  openblas/0.3.18  spglib/1.16.0
 export CC=mpicc CXX=mpicxx FC=mpifort
 module load elpa/2020.11.001  fftw/3.3.8  libint/2.6.0  plumed/2.6.2  scalapack-clang/2.1.0
-export PATH=${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-clang:$PATH
+export PATH=${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-clang-sve:$PATH
 
 [APP]
 app_name = CP2K
 build_dir = ${JARVIS_ROOT}
-binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-clang/
+binary_dir = ${JARVIS_ROOT}/cp2k-8.2/exe/Linux-arm-clang-sve/
 case_dir = ${JARVIS_ROOT}/cp2k-8.2/benchmarks/QS/
 
 [BUILD]
@@ -86,26 +86,26 @@ nvidia-smi -ac 1215,1410
 
 echo 3 > /proc/sys/vm/drop_caches
 echo "===run 32C*GPU===" >> $logfile
-mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang-sve/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
 
 echo 3 > /proc/sys/vm/drop_caches
 echo "===run 32C*2GPU===" >> $logfile
-mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang-sve/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
 
 
 echo 3 > /proc/sys/vm/drop_caches
 echo "===run 64C*GPU===" >> $logfile
-mpirun -np 64 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+mpirun -np 64 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang-sve/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
 
 echo 3 > /proc/sys/vm/drop_caches
 echo "===run 64C*2GPU===" >> $logfile
-mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+mpirun -np 32 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang-sve/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
 
 
 echo 3 > /proc/sys/vm/drop_caches
 echo "===run 128C*GPU===" >> $logfile
-mpirun -np 128 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+mpirun -np 128 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0 exe/Linux-arm-clang-sve/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
 
 echo 3 > /proc/sys/vm/drop_caches
 echo "===run 128C*2GPU===" >> $logfile
-mpirun -np 128 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
+mpirun -np 128 -genv OMP_NUM_THREADS=1 -genv CUDA_VISIBLE_DEVICES=0,1 exe/Linux-arm-clang-sve/cp2k.psmp benchmarks/QS/H2O-256.inp > cp2k.H2O-256.inp.log  >> $logfile 2>&1
-- 
Gitee