From 57b8a8ff68d73202d2cc4a9bd3544567901f9057 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Mon, 19 Aug 2024 10:17:37 +0800 Subject: [PATCH] add impl for nri and bugfix Signed-off-by: zhongtao (cherry picked from commit d31d63471c628d967571108f37fb62febd0878ce) --- ...-default-registry-mirrors-in-ci-test.patch | 26 + ...imestamp-in-PodSandboxStatu-response.patch | 25 + 0111-bugfix-for-file-param-verify.patch | 75 + 0112-bugfix-change-cni-log-info.patch | 26 + ...ve-shutdown-handle-after-init-module.patch | 43 + 0114-bugfix-for-null-pointer-reference.patch | 56 + 0115-bugfix-for-m_criService-shutdown.patch | 33 + 0116-fix-bug-in-ci-test.patch | 54 + ...RI-add-nri-head-file-and-common-func.patch | 1051 ++++ ...-plugin-cleanup-when-network-namespa.patch | 51 + ...i-add-convert-and-utils-impl-for-nri.patch | 2254 ++++++++ ...th-before-ns-mountpoint-verification.patch | 72 + ...nri-impl-for-nri-plugin-and-adaption.patch | 4724 +++++++++++++++++ iSulad.spec | 21 +- 14 files changed, 8510 insertions(+), 1 deletion(-) create mode 100644 0109-modify-default-registry-mirrors-in-ci-test.patch create mode 100644 0110-add-timestamp-in-PodSandboxStatu-response.patch create mode 100644 0111-bugfix-for-file-param-verify.patch create mode 100644 0112-bugfix-change-cni-log-info.patch create mode 100644 0113-move-shutdown-handle-after-init-module.patch create mode 100644 0114-bugfix-for-null-pointer-reference.patch create mode 100644 0115-bugfix-for-m_criService-shutdown.patch create mode 100644 0116-fix-bug-in-ci-test.patch create mode 100644 0117-NRI-add-nri-head-file-and-common-func.patch create mode 100644 0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch create mode 100644 0119-nri-add-convert-and-utils-impl-for-nri.patch create mode 100644 0120-get-realpath-before-ns-mountpoint-verification.patch create mode 100644 0121-nri-impl-for-nri-plugin-and-adaption.patch diff --git a/0109-modify-default-registry-mirrors-in-ci-test.patch b/0109-modify-default-registry-mirrors-in-ci-test.patch new file mode 100644 index 0000000..f0c1a98 --- /dev/null +++ b/0109-modify-default-registry-mirrors-in-ci-test.patch @@ -0,0 +1,26 @@ +From d6284e5e786e1407c2ce5ef098a39c154650bd38 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 12 Jun 2024 10:57:39 +0800 +Subject: [PATCH 109/121] modify default registry mirrors in ci test + +Signed-off-by: zhongtao +--- + CI/test_cases/container_cases/test_data/daemon.json | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index 2664c6b2..ab7d0360 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -24,7 +24,7 @@ + "overlay2.override_kernel_check=true" + ], + "registry-mirrors": [ +- "docker.io" ++ "https://3laho3y3.mirror.aliyuncs.com" + ], + "insecure-registries": [ + ], +-- +2.25.1 + diff --git a/0110-add-timestamp-in-PodSandboxStatu-response.patch b/0110-add-timestamp-in-PodSandboxStatu-response.patch new file mode 100644 index 0000000..2d2b276 --- /dev/null +++ b/0110-add-timestamp-in-PodSandboxStatu-response.patch @@ -0,0 +1,25 @@ +From 5087d7501308660970aa9e7c12cf5be7a3d9b063 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Wed, 12 Jun 2024 15:20:17 +0000 +Subject: [PATCH 110/121] add timestamp in PodSandboxStatu response + +Signed-off-by: jikai +--- + src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index fa726e2c..2a458a6d 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -910,6 +910,7 @@ void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, + for (auto &containerStatus : containerStatuses) { + *(reply->add_containers_statuses()) = *containerStatus; + } ++ reply->set_timestamp(util_get_now_time_nanos()); + return; + } + +-- +2.25.1 + diff --git a/0111-bugfix-for-file-param-verify.patch b/0111-bugfix-for-file-param-verify.patch new file mode 100644 index 0000000..2903c4b --- /dev/null +++ b/0111-bugfix-for-file-param-verify.patch @@ -0,0 +1,75 @@ +From d0fd2c2bf87d7befaa8810a70d7eb2061664f02f Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 14 Jun 2024 09:55:28 +0800 +Subject: [PATCH 111/121] bugfix for file param verify + +Signed-off-by: zhongtao +--- + src/cmd/isula/base/create.c | 19 +++++++++++++++++++ + src/cmd/isula/images/load.c | 6 ++++++ + 2 files changed, 25 insertions(+) + +diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c +index 543b8fd6..b04dddb5 100644 +--- a/src/cmd/isula/base/create.c ++++ b/src/cmd/isula/base/create.c +@@ -292,6 +292,12 @@ static int append_env_variables_to_conf(const char *env_file, isula_container_co + int ret = 0; + size_t file_size; + ++ if (util_dir_exists(env_file)) { ++ COMMAND_ERROR("Env file is a directory: %s", env_file); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(env_file)) { + COMMAND_ERROR("env file not exists: %s", env_file); + ret = -1; +@@ -427,6 +433,12 @@ static int append_labels_to_conf(const char *label_file, isula_container_config_ + int ret = 0; + size_t file_size; + ++ if (util_dir_exists(label_file)) { ++ COMMAND_ERROR("Label file is a directory: %s", label_file); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(label_file)) { + COMMAND_ERROR("label file not exists: %s", label_file); + ret = -1; +@@ -2357,6 +2369,13 @@ static int create_check_env_target_file(const struct client_arguments *args) + ret = -1; + goto out; + } ++ ++ if (util_dir_exists(env_path)) { ++ COMMAND_ERROR("Env target file is a directory: %s", env_path); ++ ret = -1; ++ goto out; ++ } ++ + if (!util_file_exists(env_path)) { + goto out; + } +diff --git a/src/cmd/isula/images/load.c b/src/cmd/isula/images/load.c +index 314e5d5e..cb39dee7 100644 +--- a/src/cmd/isula/images/load.c ++++ b/src/cmd/isula/images/load.c +@@ -162,6 +162,12 @@ int cmd_load_main(int argc, const char **argv) + g_cmd_load_args.file = file; + } + ++ if (util_dir_exists(g_cmd_load_args.file)) { ++ COMMAND_ERROR("Load file is a directory: %s", g_cmd_load_args.file); ++ ret = -1; ++ exit(exit_code); ++ } ++ + if (!util_file_exists(g_cmd_load_args.file)) { + COMMAND_ERROR("File %s is not exist", g_cmd_load_args.file); + exit(exit_code); +-- +2.25.1 + diff --git a/0112-bugfix-change-cni-log-info.patch b/0112-bugfix-change-cni-log-info.patch new file mode 100644 index 0000000..adc18b1 --- /dev/null +++ b/0112-bugfix-change-cni-log-info.patch @@ -0,0 +1,26 @@ +From 359a6673e01bef937adcc17f99ee94b67caca32e Mon Sep 17 00:00:00 2001 +From: liuxu +Date: Fri, 14 Jun 2024 17:12:58 +0800 +Subject: [PATCH 112/121] bugfix:change cni log info + +Signed-off-by: liuxu +--- + .../modules/network/cni_operator/libcni/invoke/libcni_exec.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +index 74d6d74a..1e4a7138 100644 +--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c ++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c +@@ -247,7 +247,7 @@ static char *env_stringify(char *(*pargs)[2], size_t len) + bool invalid_arg = (pargs == NULL || len == 0); + + if (invalid_arg) { +- ERROR("Invalid arguments"); ++ DEBUG("Empty arguments"); + return NULL; + } + +-- +2.25.1 + diff --git a/0113-move-shutdown-handle-after-init-module.patch b/0113-move-shutdown-handle-after-init-module.patch new file mode 100644 index 0000000..5ac3acc --- /dev/null +++ b/0113-move-shutdown-handle-after-init-module.patch @@ -0,0 +1,43 @@ +From 0ae6244c6bfed229a46d300888977a4967e1d718 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 19 Jun 2024 09:50:51 +0800 +Subject: [PATCH 113/121] move shutdown handle after init module + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 3e2249d7..52ac3172 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -1669,11 +1669,6 @@ static int start_daemon_threads() + { + int ret = -1; + +- if (new_shutdown_handler()) { +- ERROR("Create new shutdown handler thread failed"); +- goto out; +- } +- + if (events_module_init() != 0) { + goto out; + } +@@ -1825,6 +1820,13 @@ int main(int argc, char **argv) + goto failure; + } + ++ // after all modules are initialized, enable the shutdown handler to ++ // prevent shutdown handler from cleaning up incompletely initialized modules. ++ if (new_shutdown_handler()) { ++ ERROR("Create new shutdown handler thread failed"); ++ goto failure; ++ } ++ + #ifdef ENABLE_PLUGIN + if (start_plugin_manager()) { + ERROR("Failed to init plugin_manager"); +-- +2.25.1 + diff --git a/0114-bugfix-for-null-pointer-reference.patch b/0114-bugfix-for-null-pointer-reference.patch new file mode 100644 index 0000000..a39467e --- /dev/null +++ b/0114-bugfix-for-null-pointer-reference.patch @@ -0,0 +1,56 @@ +From 701180b53d1c52376f753b94c5cf09987ae789b3 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 18 Jun 2024 16:02:25 +0800 +Subject: [PATCH 114/121] bugfix for null pointer reference + +Signed-off-by: zhongtao +--- + src/daemon/entry/connect/grpc/grpc_service.cc | 14 ++++++++++++-- + 1 file changed, 12 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 1d8de922..300af082 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -100,7 +100,9 @@ public: + { + // Wait for the server to shutdown. Note that some other thread must be + // responsible for shutting down the server for this call to ever return. +- m_server->Wait(); ++ if (m_server != nullptr) { ++ m_server->Wait(); ++ } + + // Wait for stream server to shutdown + m_criService.Wait(); +@@ -109,7 +111,9 @@ public: + void Shutdown(void) + { + // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit +- m_criService.Shutdown(); ++ if (m_server != nullptr) { ++ m_server->Shutdown(); ++ } + + m_server->Shutdown(); + +@@ -242,10 +246,16 @@ int grpc_server_init(const struct service_arguments *args) + + void grpc_server_wait(void) + { ++ if (g_grpcserver == nullptr) { ++ return; ++ } + g_grpcserver->Wait(); + } + + void grpc_server_shutdown(void) + { ++ if (g_grpcserver == nullptr) { ++ return; ++ } + g_grpcserver->Shutdown(); + } +-- +2.25.1 + diff --git a/0115-bugfix-for-m_criService-shutdown.patch b/0115-bugfix-for-m_criService-shutdown.patch new file mode 100644 index 0000000..8d4d859 --- /dev/null +++ b/0115-bugfix-for-m_criService-shutdown.patch @@ -0,0 +1,33 @@ +From 93b1df1a1d3fcf6d285102f3cc1f79e6241aa393 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 4 Jul 2024 10:58:38 +0800 +Subject: [PATCH 115/121] bugfix for m_criService shutdown + +Signed-off-by: zhongtao +--- + src/daemon/entry/connect/grpc/grpc_service.cc | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 300af082..fb5ec3cb 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -111,12 +111,12 @@ public: + void Shutdown(void) + { + // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit ++ m_criService.Shutdown(); ++ + if (m_server != nullptr) { + m_server->Shutdown(); + } +- +- m_server->Shutdown(); +- ++ + // Shutdown daemon, this operation should remove socket file. + for (const auto &address : m_socketPath) { + if (address.find(UNIX_SOCKET_PREFIX) == 0) { +-- +2.25.1 + diff --git a/0116-fix-bug-in-ci-test.patch b/0116-fix-bug-in-ci-test.patch new file mode 100644 index 0000000..181fbc7 --- /dev/null +++ b/0116-fix-bug-in-ci-test.patch @@ -0,0 +1,54 @@ +From c7cf33c432b3d9479b2fe365169d4b9a37cae8f7 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 9 Jul 2024 12:30:01 +0000 +Subject: [PATCH 116/121] fix bug in ci test + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/run.sh | 6 +++--- + CI/test_cases/helpers.sh | 5 +++-- + 2 files changed, 6 insertions(+), 5 deletions(-) + +diff --git a/CI/test_cases/container_cases/run.sh b/CI/test_cases/container_cases/run.sh +index 1bfd388b..ef04b547 100755 +--- a/CI/test_cases/container_cases/run.sh ++++ b/CI/test_cases/container_cases/run.sh +@@ -26,9 +26,9 @@ source ../helpers.sh + function do_test_t() + { + tid=`isula run --runtime $1 -tid --name hostname busybox` +- chostname=`isula exec -it $tid hostname` +- clean_hostname=$(echo "$hostname" | sed 's/[\x01-\x1F\x7F]//g') +- fn_check_eq "${clean_hostname}" "${tid:0:12}" "default hostname is not id of container" ++ # should not use -it option, otherwise the hostname will containe special characters such as '$' or '\r' ++ hostname=`isula exec $tid hostname` ++ fn_check_eq "${hostname}" "${tid:0:12}" "default hostname is not id of container" + isula exec -it hostname env | grep HOSTNAME + fn_check_eq "$?" "0" "check HOSTNAME env failed" + isula stop -t 0 $tid +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index c5eba8a2..0288b4ea 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -52,15 +52,16 @@ function cut_output_lines() { + return $retval + } + ++# use string compare to check the result + function fn_check_eq() { +- if [[ "$1" -ne "$2" ]];then ++ if [ "x$1" != "x$2" ];then + echo "$3" + TC_RET_T=$(($TC_RET_T+1)) + fi + } + + function fn_check_ne() { +- if [[ "$1" -eq "$2" ]];then ++ if [[ "x$1" == "x$2" ]];then + echo "$3" + TC_RET_T=$(($TC_RET_T+1)) + fi +-- +2.25.1 + diff --git a/0117-NRI-add-nri-head-file-and-common-func.patch b/0117-NRI-add-nri-head-file-and-common-func.patch new file mode 100644 index 0000000..61c287e --- /dev/null +++ b/0117-NRI-add-nri-head-file-and-common-func.patch @@ -0,0 +1,1051 @@ +From d21dc88cb33049ec5d9ce8e16eb1a2f9c070219b Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 30 Jul 2024 20:17:08 +0800 +Subject: [PATCH 117/121] [NRI] add nri head file and common func + +Signed-off-by: zhongtao +--- + cmake/checker.cmake | 8 ++ + cmake/options.cmake | 13 ++- + src/CMakeLists.txt | 10 +++ + src/daemon/CMakeLists.txt | 9 ++ + src/daemon/common/CMakeLists.txt | 5 ++ + src/daemon/common/nri/CMakeLists.txt | 13 +++ + src/daemon/common/nri/nri_convert.h | 36 ++++++++ + src/daemon/common/nri/nri_spec.h | 24 ++++++ + src/daemon/common/nri/nri_utils.h | 71 ++++++++++++++++ + src/daemon/config/isulad_config.h | 16 ++++ + src/daemon/modules/api/specs_api.h | 10 ++- + src/daemon/modules/spec/specs.c | 67 ++++++++++++++- + src/daemon/nri/CMakeLists.txt | 6 ++ + src/daemon/nri/nri_adaption.h | 123 +++++++++++++++++++++++++++ + src/daemon/nri/nri_helpers.h | 63 ++++++++++++++ + src/daemon/nri/nri_plugin_ops.h | 38 +++++++++ + src/daemon/nri/nri_result.h | 114 +++++++++++++++++++++++++ + src/daemon/nri/plugin.h | 95 +++++++++++++++++++++ + src/utils/cpputils/transform.cc | 16 ++++ + src/utils/cpputils/transform.h | 2 + + src/utils/cutils/utils.c | 7 ++ + src/utils/cutils/utils.h | 2 + + src/utils/tar/util_archive.c | 7 -- + 23 files changed, 744 insertions(+), 11 deletions(-) + create mode 100644 src/daemon/common/nri/CMakeLists.txt + create mode 100644 src/daemon/common/nri/nri_convert.h + create mode 100644 src/daemon/common/nri/nri_spec.h + create mode 100644 src/daemon/common/nri/nri_utils.h + create mode 100644 src/daemon/nri/CMakeLists.txt + create mode 100644 src/daemon/nri/nri_adaption.h + create mode 100644 src/daemon/nri/nri_helpers.h + create mode 100644 src/daemon/nri/nri_plugin_ops.h + create mode 100644 src/daemon/nri/nri_result.h + create mode 100644 src/daemon/nri/plugin.h + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index 13f0fd62..b877c1c8 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -113,6 +113,14 @@ if (ENABLE_SHIM_V2) + _CHECK(LIBSHIM_V2_LIBRARY "LIBSHIM_V2_LIBRARY-NOTFOUND" "libshim_v2.so") + endif() + ++if (ENABLE_NRI) ++ find_path(NRI_INCLUDE_DIR nri_plugin.h) ++ _CHECK(NRI_INCLUDE_DIR "NRI_INCLUDE_DIR-NOTFOUND" "nri_plugin.h") ++ ++ find_library(LIBISULA_NRI_LIBRARY isula_nri) ++ _CHECK(LIBISULA_NRI_LIBRARY "LIBISULA_NRI_LIBRARY-NOTFOUND" "libisula_nri.so") ++endif() ++ + if (OPENSSL_VERIFY) + find_path(OPENSSL_INCLUDE_DIR openssl/x509.h) + _CHECK(OPENSSL_INCLUDE_DIR "OPENSSL_INCLUDE_DIR-NOTFOUND" "openssl/x509.h") +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 5b17f631..41177fe0 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -59,12 +59,23 @@ if (ENABLE_SANDBOXER STREQUAL "ON") + endif() + + option(ENABLE_OOM_MONITOR "Enable oom monitor" ON) +-IF (ENABLE_OOM_MONITOR STREQUAL "ON") ++if (ENABLE_OOM_MONITOR STREQUAL "ON") + add_definitions(-DENABLE_OOM_MONITOR) + set(ENABLE_OOM_MONITOR 1) + message("${Green}-- Enable oom monitor${ColourReset}") + endif() + ++option(ENABLE_NRI "Enable NRI API" OFF) ++if (ENABLE_NRI STREQUAL "ON") ++ if (ENABLE_CRI_API_V1) ++ add_definitions(-DENABLE_NRI) ++ set(ENABLE_NRI 1) ++ message("${Green}-- Enable NRI API${ColourReset}") ++ else() ++ message("${Yellow}-- Can not enable NRI, NRI need enable CRI API V1 first ${ColourReset}") ++ endif() ++endif() ++ + option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON) + if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON") + add_definitions(-DSYSTEMD_NOTIFY) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 48c1bad0..70a8ac91 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -15,6 +15,12 @@ if (ENABLE_SHIM_V2) + ) + endif() + ++if (ENABLE_NRI) ++ list(APPEND CHECKED_INCLUDE_DIRS ++ ${NRI_INCLUDE_DIR} ++ ) ++endif() ++ + if (GRPC_CONNECTOR) + list(APPEND CHECKED_INCLUDE_DIRS + ${GRPC_INCLUDE_DIR} +@@ -168,6 +174,10 @@ if (ENABLE_SHIM_V2) + target_link_libraries(isulad ${LIBSHIM_V2_LIBRARY}) + endif() + ++if (ENABLE_NRI) ++ target_link_libraries(isulad ${LIBISULA_NRI_LIBRARY}) ++endif() ++ + if (ENABLE_EMBEDDED_IMAGE) + target_link_libraries(isulad ${SQLITE3_LIBRARY}) + endif() +diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt +index 29af3dca..801f5797 100644 +--- a/src/daemon/CMakeLists.txt ++++ b/src/daemon/CMakeLists.txt +@@ -36,6 +36,15 @@ if (ENABLE_CRI_API_V1) + list (APPEND local_daemon_incs + ${SANDBOX_INCS} + ) ++ if (ENABLE_NRI) ++ add_subdirectory(nri) ++ list (APPEND local_daemon_srcs ++ ${NRI_SRCS} ++ ) ++ list (APPEND local_daemon_incs ++ ${NRI_INCS} ++ ) ++ endif() + endif() + + set(DAEMON_SRCS +diff --git a/src/daemon/common/CMakeLists.txt b/src/daemon/common/CMakeLists.txt +index e88578dd..ffeed4b3 100644 +--- a/src/daemon/common/CMakeLists.txt ++++ b/src/daemon/common/CMakeLists.txt +@@ -10,12 +10,16 @@ if (GRPC_CONNECTOR) + endif() + + add_subdirectory(cgroup) ++if (ENABLE_NRI) ++ add_subdirectory(nri) ++endif() + + set(local_daemon_common_srcs ${daemon_common_top_srcs}) + + set(DAEMON_COMMON_SRCS + ${COMMON_CRI_SRCS} + ${COMMON_CGROUP_SRCS} ++ ${COMMON_NRI_SRCS} + ${local_daemon_common_srcs} + PARENT_SCOPE + ) +@@ -23,6 +27,7 @@ set(DAEMON_COMMON_SRCS + set(DAEMON_COMMON_INCS + ${COMMON_CRI_INCS} + ${COMMON_CGROUP_INCS} ++ ${COMMON_NRI_INCS} + ${CMAKE_CURRENT_SOURCE_DIR} + PARENT_SCOPE + ) +diff --git a/src/daemon/common/nri/CMakeLists.txt b/src/daemon/common/nri/CMakeLists.txt +new file mode 100644 +index 00000000..512ba694 +--- /dev/null ++++ b/src/daemon/common/nri/CMakeLists.txt +@@ -0,0 +1,13 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_common_nri_srcs) ++set(local_common_nri_incs ${CMAKE_CURRENT_SOURCE_DIR}) ++ ++set(COMMON_NRI_SRCS ++ ${local_common_nri_srcs} ++ PARENT_SCOPE ++) ++ ++set(COMMON_NRI_INCS ++ ${local_common_nri_incs} ++ PARENT_SCOPE ++) +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +new file mode 100644 +index 00000000..883f7c41 +--- /dev/null ++++ b/src/daemon/common/nri/nri_convert.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-16 ++ * Description: provide nri convert functions ++ *********************************************************************************/ ++#ifndef DAEMON_COMMON_NRI_NRI_CONVERT_H ++#define DAEMON_COMMON_NRI_NRI_CONVERT_H ++#include ++#include ++#include ++#include ++ ++#include "isula_libutils/nri_pod_sandbox.h" ++#include "isula_libutils/nri_container.h" ++#include "isula_libutils/nri_mount.h" ++#include "isula_libutils/nri_linux_resources.h" ++ ++#include "sandbox.h" ++#include "api_v1.pb.h" ++ ++auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox *pod) -> bool; ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; ++auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; ++auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; ++ ++auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; ++#endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h +new file mode 100644 +index 00000000..e7c5035d +--- /dev/null ++++ b/src/daemon/common/nri/nri_spec.h +@@ -0,0 +1,24 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri oci functions ++ *********************************************************************************/ ++ ++#ifndef DAEMON_COMMON_NRI_NRI_SPEC_H ++#define DAEMON_COMMON_NRI_NRI_SPEC_H ++ ++#include ++#include ++ ++int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); ++ ++#endif // DAEMON_COMMON_NRI_NRI_SPEC_H +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +new file mode 100644 +index 00000000..3aa50ae4 +--- /dev/null ++++ b/src/daemon/common/nri/nri_utils.h +@@ -0,0 +1,71 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri utils functions ++ *********************************************************************************/ ++#ifndef DAEMON_COMMON_NRI_NRI_UTILS_H ++#define DAEMON_COMMON_NRI_NRI_UTILS_H ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef enum { ++ UNKNOWN = 0, ++ RUN_POD_SANDBOX = 1, ++ STOP_POD_SANDBOX = 2, ++ REMOVE_POD_SANDBOX = 3, ++ CREATE_CONTAINER = 4, ++ POST_CREATE_CONTAINER = 5, ++ START_CONTAINER = 6, ++ POST_START_CONTAINER = 7, ++ UPDATE_CONTAINER = 8, ++ POST_UPDATE_CONTAINER = 9, ++ STOP_CONTAINER = 10, ++ REMOVE_CONTAINER = 11, ++ LAST = 12, ++} NRI_Event; ++ ++bool copy_nri_mount(const nri_mount *src, nri_mount **dest); ++bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); ++bool copy_nri_hook(const nri_hook *src, nri_hook **dest); ++bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); ++bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); ++bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); ++ ++bool is_marked_for_removal(const char* key, char **out); ++ ++bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); ++ ++bool init_nri_container_adjust(nri_container_adjustment **adjust); ++bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); ++bool init_nri_linux_resources(nri_linux_resources **resources); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // DAEMON_COMMON_NRI_NRI_UTILS_H +\ No newline at end of file +diff --git a/src/daemon/config/isulad_config.h b/src/daemon/config/isulad_config.h +index f29cd564..c57ca8d6 100644 +--- a/src/daemon/config/isulad_config.h ++++ b/src/daemon/config/isulad_config.h +@@ -40,6 +40,22 @@ struct isulad_conf { + #define DEFAULT_SANDBOXER_NAME "shim" + char *conf_get_sandbox_rootpath(void); + char *conf_get_sandbox_statepath(void); ++ ++#ifdef ENABLE_NRI ++#define DEFAULT_SOCKET_PATH "/var/run/nri/nri.sock" ++#define DEFAULT_MUX_SOCKET_PATH "/var/run/nri/nri_mux.sock" ++#define DEFAULT_PLUGIN_PATH "/opt/nri/plugins" ++#define DEFAULT_PLUGIN_CONFIG_PATH "/etc/nri/conf.d" ++#define DEFAULT_PLUGIN_REGISTRY_TIMEOUT 5 ++#define DEFAULT_PLUGIN_REQUST_TIMEOUT 2 ++bool conf_get_nri_support(void); ++bool conf_get_nri_external_support(void); ++char *conf_get_nri_plugin_config_path(void); ++char *conf_get_nri_plugin_path(void); ++char *conf_get_socket_path(void); ++uint64_t conf_get_nri_plugin_registration_timeout(void); ++uint64_t conf_get_nri_plugin_requst_timeout(void); ++#endif + #endif + + char *conf_get_isulad_pidfile(void); +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 0999836b..6a1cd776 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -58,6 +58,9 @@ int spec_module_init(void); + #ifdef ENABLE_CDI + int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len); + int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, size_t env_len); ++#endif/* ENABLE_CDI */ ++ ++#if defined (ENABLE_CDI) || defined(ENABLE_NRI) + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device); + int spec_add_linux_resources_device(oci_runtime_spec *oci_spec, bool allow, const char *dev_type, + int64_t major, int64_t minor, const char *access); +@@ -66,7 +69,12 @@ int spec_add_mount(oci_runtime_spec *oci_spec, defs_mount *mnt); + int spec_add_prestart_hook(oci_runtime_spec *oci_spec, defs_hook *prestart_hook); + int spec_add_poststart_hook(oci_runtime_spec *oci_spec, defs_hook *poststart_hook); + int spec_add_poststop_hook(oci_runtime_spec *oci_spec, defs_hook *poststop_hook); +-#endif /* ENABLE_CDI */ ++#endif /* ENABLE_CDI || ENABLE_NRI */ ++ ++#ifdef ENABLE_NRI ++int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit); ++int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); ++#endif /* ENABLE_NRI */ + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index f0538e26..1fd9e5a8 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2608,7 +2608,7 @@ int spec_module_init(void) + static int add_env(defs_process *dp, const char *env, const char *key) + { + size_t i; +- ++ + for (i = 0; i < dp->env_len; i++) { + __isula_auto_free char *oci_key = NULL; + if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) { +@@ -2684,7 +2684,9 @@ int spec_add_multiple_process_env(oci_runtime_spec *oci_spec, const char **envs, + + return ret; + } ++#endif /* ENABLE_CDI */ + ++#if defined (ENABLE_CDI) || defined(ENABLE_NRI) + int spec_add_device(oci_runtime_spec *oci_spec, defs_device *device) + { + int ret = 0; +@@ -2817,4 +2819,65 @@ SPEC_ADD_HOOKS_ITEM_DEF(prestart) + SPEC_ADD_HOOKS_ITEM_DEF(poststart) + SPEC_ADD_HOOKS_ITEM_DEF(poststop) + +-#endif /* ENABLE_CDI */ +\ No newline at end of file ++#endif /* ENABLE_CDI || ENABLE_NRI */ ++ ++#ifdef ENABLE_NRI ++int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const char *page_size, uint64_t limit) ++{ ++ int ret = 0; ++ defs_resources_hugepage_limits_element *hugepage_limit = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ hugepage_limit = util_common_calloc_s(sizeof(*hugepage_limit)); ++ if (hugepage_limit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ hugepage_limit->page_size = util_strdup_s(page_size); ++ hugepage_limit->limit = limit; ++ ++ if (util_mem_realloc((void **)&oci_spec->linux->resources->hugepage_limits, ++ (oci_spec->linux->resources->hugepage_limits_len + 1) * sizeof(defs_resources_hugepage_limits_element *), ++ (void *)oci_spec->linux->resources->hugepage_limits, ++ oci_spec->linux->resources->hugepage_limits_len * sizeof(defs_resources_hugepage_limits_element *)) != 0) { ++ ERROR("Out of memory"); ++ free_defs_resources_hugepage_limits_element(hugepage_limit); ++ return -1; ++ } ++ oci_spec->linux->resources->hugepage_limits[oci_spec->linux->resources->hugepage_limits_len] = hugepage_limit; ++ oci_spec->linux->resources->hugepage_limits_len++; ++ ++ return 0; ++} ++ ++int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft) ++{ ++ int ret = 0; ++ defs_process_rlimits_element *rlimit = NULL; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ rlimit = util_common_calloc_s(sizeof(*rlimit)); ++ if (rlimit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ rlimit->type = util_strdup_s(type); ++ rlimit->hard = hard; ++ rlimit->soft = soft; ++ ++ if (do_merge_one_ulimit_override(oci_spec, rlimit) != 0) { ++ ERROR("Failed to merge one nri ulimit to oci spec"); ++ return -1; ++ } ++ ++ return 0; ++} ++#endif /* ENABLE_NRI */ +\ No newline at end of file +diff --git a/src/daemon/nri/CMakeLists.txt b/src/daemon/nri/CMakeLists.txt +new file mode 100644 +index 00000000..361d79f8 +--- /dev/null ++++ b/src/daemon/nri/CMakeLists.txt +@@ -0,0 +1,6 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} nri_top_srcs) ++ ++set(NRI_INCS ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) ++ ++set(NRI_SRCS ${nri_top_srcs} PARENT_SCOPE) +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +new file mode 100644 +index 00000000..7f0640df +--- /dev/null ++++ b/src/daemon/nri/nri_adaption.h +@@ -0,0 +1,123 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin manager(NRI adaption) class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_NRI_ADAPTION_H ++#define DAEMON_NRI_PLUGIN_NRI_ADAPTION_H ++ ++// #include "read_write_lock.h" ++#include ++#include ++ ++#include "plugin.h" ++#include "sandbox.h" ++#include "v1_cri_container_manager_service.h" ++ ++const std::string PluginNameEnv = "NRI_PLUGIN_NAME"; ++const std::string PluginIdxEnv = "NRI_PLUGIN_IDX"; ++const std::string PluginSocketEnv = "NRI_PLUGIN_SOCKET"; ++ ++struct nri_plugin_exec_args_t { ++ const char *workdir; ++ const char *cmd; ++ const char *name; ++ const char *index; ++ const uint32_t sockFd; ++}; ++ ++class NRIAdaptation { ++public: ++ // Singleton ++ static NRIAdaptation *GetInstance() noexcept; ++ ++ // initialize value ++ auto Init(Errors &error) -> bool; ++ ++ auto GetSockpath(std::vector &paths) -> bool; ++ ++ // Stop plugins. ++ auto StopPlugins() -> bool; ++ ++ void RemoveClosedPlugins(); ++ ++ auto GetPluginByIndex(const std::string &index) -> std::shared_ptr; ++ void AddPluginByIndex(const std::string &index, std::shared_ptr plugin); ++ void RemovePluginByIndex(const std::string &index); ++ ++ auto RunPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; ++ auto StopPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; ++ auto RemovePodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; ++ auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; ++ auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; ++ auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, Errors &error) -> bool; ++ auto StartContainer(const std::string &conId, Errors &error) ->bool; ++ auto PostStartContainer(const std::string &conId, Errors &error) ->bool; ++ auto UpdateContainer(const std::string &conId, Errors &error) ->bool; ++ auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; ++ auto StopContainer(const std::string &conId, Errors &error) ->bool; ++ auto RemoveContainer(const std::string &conId, Errors &error) ->bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; ++ auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; ++ ++ auto NewExternalPlugin(int fd) -> bool; ++private: ++ NRIAdaptation() = default; ++ NRIAdaptation(const NRIAdaptation &other) = delete; ++ NRIAdaptation &operator=(const NRIAdaptation &) = delete; ++ virtual ~NRIAdaptation(); ++ ++ auto StartPlugin() -> bool; ++ auto NewLaunchedPlugin(const std::shared_ptr &) -> bool; ++ auto DiscoverPlugins(std::map> &map) -> bool; ++ // Synchronizing NRI (plugin) with current runtime state ++ auto SyncPlugin() -> bool; ++ ++ auto SortPlugins() -> bool; ++ void GetClosedPlugins(std::vector &closedPlugin); ++ ++ auto IsSupport() -> bool; ++ ++ auto ApplyUpdates(const std::vector &update, std::vector &failed, bool getFailed, ++ Errors &error) -> bool; ++ ++ auto NRIPodSandbox(const std::shared_ptr &sandbox, Errors& error) -> std::unique_ptr>; ++ auto NRIContainerByConConfig(const std::shared_ptr &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr>; ++ auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr>; ++ ++ auto GetNRIPluginConfigPath(void) -> std::string; ++ auto GetNRIPluginPath(void) -> std::string; ++ auto GetNRISockPath(void) -> std::string; ++private: ++ RWMutex m_mutex; ++ static std::atomic m_instance; ++ bool m_support; ++ bool m_external_support; ++ std::string m_version; ++ std::string m_sock_path; ++ std::string m_pluginConfigPath; ++ std::string m_pluginPath; ++ std::vector m_socketPathArr; ++ std::string m_disableConnections; ++ // id --> NRIPlugin map ++ std::map> m_storeMap; ++ // TODO:plugin monitor thread id?? ++ // shutdown() to clean resource ++ // init to create thread ++ // todo: if Singleton? ++ std::unique_ptr m_containerManager; ++ int64_t m_plugin_registration_timeout; ++ int64_t m_plugin_requst_timeout; ++}; ++ ++#endif // DAEMON_NRI_PLUGIN_NRI_ADAPTION_H +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +new file mode 100644 +index 00000000..06ee8419 +--- /dev/null ++++ b/src/daemon/nri/nri_helpers.h +@@ -0,0 +1,63 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2017-2019. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-13 ++ * Description: provide nri helpers functions ++ *********************************************************************************/ ++#ifndef DAEMON_NRI_PLUGIN_NRI_HELPERS_H ++#define DAEMON_NRI_PLUGIN_NRI_HELPERS_H ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "errors.h" ++ ++namespace NRIHelpers { ++std::string MarkForRemoval(const std::string &key); ++ ++auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; ++ ++std::string GenerateRandomExternalName(void); ++ ++bool CheckPluginIndex(const std::string &idx); ++ ++template ++void freeArray(T ** &arr, int size) ++{ ++ if (arr == NULL) { ++ return; ++ } ++ ++ for (int i = 0; i < size; i++) { ++ if (arr[i] == NULL) { ++ return; ++ } ++ free(arr[i]); ++ } ++ ++ free(arr); ++ arr = NULL; ++} ++}; // namespace NRIHelpers ++ ++#endif // DAEMON_NRI_PLUGIN_NRI_HELPERS_H +diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h +new file mode 100644 +index 00000000..37d437d2 +--- /dev/null ++++ b/src/daemon/nri/nri_plugin_ops.h +@@ -0,0 +1,38 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtaoo ++ * Create: 2024-03-26 ++ * Description: provide nri plugin api definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_OPS_H ++#define DAEMON_NRI_PLUGIN_OPS_H ++ ++#include ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++bool nri_adaption_init(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); ++int nri_registry_containers(const nri_register_plugin_request *request); ++ ++int nri_external_plugin_connect(int fd); ++ ++#endif // DAEMON_NRI_PLUGIN_OPS_H +diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h +new file mode 100644 +index 00000000..f2896ea0 +--- /dev/null ++++ b/src/daemon/nri/nri_result.h +@@ -0,0 +1,114 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-06-29 ++ * Description: provide nri result definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_NRI_RESULT_H ++#define DAEMON_NRI_PLUGIN_NRI_RESULT_H ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "api_v1.pb.h" ++#include "nri_helpers.h" ++#include "nri_utils.h" ++ ++using EventMask = std::int32_t; ++ ++const EventMask ValidEvents = (1 << (LAST - 1)) - 1; ++ ++struct owners { ++ std::map annotations; ++ std::map mounts; ++ std::map devices; ++ std::map env; ++ std::string memLimit; ++ std::string memReservation; ++ std::string memSwapLimit; ++ std::string memKernelLimit; ++ std::string memTCPLimit; ++ std::string memSwappiness; ++ std::string memDisableOomKiller; ++ std::string memUseHierarchy; ++ std::string cpuShares; ++ std::string cpuQuota; ++ std::string cpuPeriod; ++ std::string cpuRealtimeRuntime; ++ std::string cpuRealtimePeriod; ++ std::string cpusetCpus; ++ std::string cpusetMems; ++ std::map hugepageLimits; ++ std::string blockioClass; ++ std::string rdtClass; ++ std::map unified; ++ std::string cgroupsPath; ++ std::map rlimits; ++}; ++ ++struct resultReply { ++ nri_container_adjustment* adjust; ++ std::vector update; ++}; ++ ++using resultOwners = std::map; ++ ++class pluginResult { ++public: ++ pluginResult() = default; ++ ++ ~pluginResult() = default; ++ ++ auto InitByConId(std::string conId) -> bool; ++ auto InitByUpdateReq(nri_update_container_request *req) -> bool; ++ ++ auto GetReplyUpdate() -> std::vector; ++ auto GetReplyAdjust() -> nri_container_adjustment *; ++ ++ auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, ++ const std::string &plugin) -> bool; ++ auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; ++ ++private: ++ auto GetContainerUpdate(nri_container_update *update, const std::string &plugin, nri_container_update **out) -> bool; ++ auto UpdateResources(nri_container_update *reply, nri_container_update *u, const std::string &plugin) -> bool; ++ ++ auto InitReply(void) -> bool; ++ ++ auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; ++ ++ auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; ++ auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; ++ auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; ++ auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; ++ auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; ++ auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; ++ bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, ++ nri_linux_resources *dest); ++ auto AdjustCgroupsPath(char *path, const std::string &plugin) -> bool; ++ auto AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool; ++ ++private: ++ std::string m_conId; ++ nri_linux_resources *m_update_req; ++ resultReply m_reply; ++ std::map m_updates; ++ resultOwners m_owners; ++}; ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h +new file mode 100644 +index 00000000..f60a9b3d +--- /dev/null ++++ b/src/daemon/nri/plugin.h +@@ -0,0 +1,95 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin class definition ++ *********************************************************************************/ ++ ++#ifndef DAEMON_NRI_PLUGIN_PLUGIN_H ++#define DAEMON_NRI_PLUGIN_PLUGIN_H ++ ++#include ++ ++#include ++#include ++#include ++ ++#include "errors.h" ++#include "read_write_lock.h" ++#include "nri_result.h" ++ ++const std::string NRIRruntime = "v2"; ++const std::string NRIVersion = "2.0.0-beta.2+unknown"; ++ ++class NRIPlugin { ++public: ++ // init client conn ++ NRIPlugin(std::string &idx, std::string &name, std::string &config); ++ NRIPlugin(int fd, std::string &name); ++ // todo: close client conn ?? or single close func? ++ virtual ~NRIPlugin() = default; ++ // wait for plugin to register, then configure it. ++ auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; ++ // close a plugin shutting down its multiplexed ttrpc connections. ++ auto Close(void) -> bool; ++ // stop a plugin (if it was launched by us) ++ auto Stop(void) -> bool; ++ ++ // Name returns a string indentication for the plugin. ++ auto GetName(void) -> const std::string &; ++ auto GetIndex(void) -> const std::string &; ++ auto GetPeerSockFd(void) -> uint32_t; ++ auto GetQualifiedName(void) -> std::string; ++ ++ void SetReady(void); ++ void SetPid(int pid); ++ ++ auto IsClose(void) -> bool; ++ ++ auto CreateSocketPair(void) -> bool; ++ ++ auto Configure(Errors &error) -> bool; ++ // Only called in external plugin scenario ++ auto Synchronize(std::vector> pods, ++ std::vector> &containers, nri_container_update ***update, size_t update_len, ++ Errors &error) -> bool; ++ auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; ++ auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; ++ auto StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, Errors &error) -> bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; ++ ++private: ++ auto Connect(int64_t timeout) -> bool; ++ ++ auto WaitForReady(int64_t timeout) -> bool; ++ auto IsSetEvent(EventMask e) -> bool; ++ ++private: ++ RWMutex m_mutex; ++ bool m_external; ++ std::string m_idx; ++ std::string m_name; ++ std::string m_config; ++ int m_pid; ++ std::string m_cmd; ++ std::vector m_sockFds; ++ std::string m_localFileName; ++ std::string m_peerFileName; ++ // TODO:zhontao monitor? ++ bool m_closed; ++ std::mutex m_readyMutex; ++ bool m_ready; ++ std::condition_variable m_condition; ++ EventMask m_events; ++}; ++ ++ ++#endif // DAEMON_NRI_PLUGIN_PLUGIN_H +\ No newline at end of file +diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc +index 74c178a9..51c154fb 100644 +--- a/src/utils/cpputils/transform.cc ++++ b/src/utils/cpputils/transform.cc +@@ -89,4 +89,20 @@ void CharArrayToStringVector(const char **src, size_t len, std::vector &ptrs) -> char ** ++{ ++ size_t len = ptrs.size(); ++ char **result = (char **)util_smart_calloc_s(sizeof(char *), (len + 1)); ++ if (result == nullptr) { ++ return nullptr; ++ } ++ size_t i {}; ++ for (const auto &it : ptrs) { ++ result[i++] = util_strdup_s(it.c_str()); ++ } ++ ++ return result; ++} ++ + } // namespace Transform +diff --git a/src/utils/cpputils/transform.h b/src/utils/cpputils/transform.h +index 476f39a6..57c58d9e 100644 +--- a/src/utils/cpputils/transform.h ++++ b/src/utils/cpputils/transform.h +@@ -35,6 +35,8 @@ auto StringVectorToCharArray(std::vector &strVec) -> char **; + + void CharArrayToStringVector(const char **src, size_t len, std::vector &dest); + ++auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField &ptrs) -> char **; ++ + }; // namespace Transform + + #endif // UTILS_CPPUTILS_TRANSFORM_H +\ No newline at end of file +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 9a33f935..f81a9141 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1699,3 +1699,10 @@ int util_chown_for_shm(const char *shm_path, const char *user_remap) + + return 0; + } ++ ++void set_child_process_pdeathsig(void) ++{ ++ if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { ++ SYSERROR("Failed to set child process pdeathsig"); ++ } ++} +\ No newline at end of file +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index ce0ca703..76684ed3 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -409,6 +409,8 @@ int util_create_shm_path(const char *spath, const int64_t shm_size); + + int util_chown_for_shm(const char *shm_path, const char *user_remap); + ++void set_child_process_pdeathsig(void); ++ + /** + * retry_cnt: max count of call cb; + * interval_us: how many us to sleep, after call cb; +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 204dab83..985e0f16 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -812,13 +812,6 @@ static void close_archive_pipes_fd(int *pipes, size_t pipe_size) + } + } + +-static void set_child_process_pdeathsig(void) +-{ +- if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { +- SYSERROR("Failed to set child process pdeathsig"); +- } +-} +- + int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options, + const char *root_dir, char **errmsg) + { +-- +2.25.1 + diff --git a/0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch b/0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch new file mode 100644 index 0000000..4b33de5 --- /dev/null +++ b/0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch @@ -0,0 +1,51 @@ +From ee720f966fdf14a99b8ebc685f3948bb8b29ba73 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 13 Aug 2024 10:56:44 +0800 +Subject: [PATCH 118/121] skip calling cni plugin cleanup when network + namespace is not mounted + +Signed-off-by: zhongtao +--- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 7 +++++++ + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 7 +++++++ + 2 files changed, 14 insertions(+) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 2a458a6d..77faf48a 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -435,6 +435,13 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptrGetSandboxConfig(); + std::map stdAnnos; + CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos); +diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +index bc3f4031..5590827e 100644 +--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc +@@ -848,6 +848,13 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + goto cleanup; + } + ++ // If the network namespace is not mounted, the network has been cleaned up ++ // and there is no need to call the cni plugin. ++ if (!util_detect_mounted(netnsPath.c_str())) { ++ WARN("Network namespace %s not exist", netnsPath.c_str()); ++ goto cleanup; ++ } ++ + stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath)); + pluginErr.Clear(); + m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, +-- +2.25.1 + diff --git a/0119-nri-add-convert-and-utils-impl-for-nri.patch b/0119-nri-add-convert-and-utils-impl-for-nri.patch new file mode 100644 index 0000000..ba76c2e --- /dev/null +++ b/0119-nri-add-convert-and-utils-impl-for-nri.patch @@ -0,0 +1,2254 @@ +From c0d4b523c24e88b8c70cd3b121a46b7b3c841c17 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 13 Aug 2024 20:33:28 +0800 +Subject: [PATCH 119/121] [nri] add convert and utils impl for nri + +Signed-off-by: zhongtao +--- + src/daemon/common/nri/nri_convert.cc | 539 ++++++++++++++++++++++++ + src/daemon/common/nri/nri_convert.h | 9 +- + src/daemon/common/nri/nri_spec.c | 602 +++++++++++++++++++++++++++ + src/daemon/common/nri/nri_utils.c | 520 +++++++++++++++++++++++ + src/daemon/common/nri/nri_utils.h | 5 +- + src/daemon/config/isulad_config.c | 178 ++++++++ + src/daemon/modules/api/specs_api.h | 5 + + src/daemon/modules/spec/specs.c | 32 +- + src/daemon/nri/nri_adaption.h | 46 +- + src/daemon/nri/nri_helpers.cc | 93 +++++ + src/daemon/nri/nri_helpers.h | 2 +- + src/utils/cpputils/transform.cc | 2 +- + 12 files changed, 2002 insertions(+), 31 deletions(-) + create mode 100644 src/daemon/common/nri/nri_convert.cc + create mode 100644 src/daemon/common/nri/nri_spec.c + create mode 100644 src/daemon/common/nri/nri_utils.c + create mode 100644 src/daemon/nri/nri_helpers.cc + +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +new file mode 100644 +index 00000000..7cce64ec +--- /dev/null ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -0,0 +1,539 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-16 ++ * Description: provide nri convert functions ++ *********************************************************************************/ ++ ++#include "nri_convert.h" ++ ++#include "container_api.h" ++#include "v1_cri_helpers.h" ++#include "path.h" ++#include "transform.h" ++#include "nri_utils.h" ++ ++static int64_t DefaultOOMScoreAdj = 0; ++ ++static bool NRILinuxCpuFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_cpu &cpu) ++{ ++ if (!config.cpuset_cpus().empty()) { ++ cpu.cpus = util_strdup_s(config.cpuset_cpus().c_str()); ++ } ++ ++ if (!config.cpuset_mems().empty()) { ++ cpu.mems = util_strdup_s(config.cpuset_mems().c_str()); ++ } ++ ++ cpu.period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.period == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.period) = config.cpu_period(); ++ ++ cpu.quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (cpu.quota == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.quota) = config.cpu_quota(); ++ ++ cpu.shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (cpu.shares == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(cpu.shares) = config.cpu_shares(); ++ ++ // consistent with other container engines, ++ // not obtained cpu.realtime_period & cpu.realtime_runtime ++ return true; ++} ++ ++static bool NRILinuxMemoryFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_memory &memory) ++{ ++ memory.limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (memory.limit == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(memory.limit) = config.memory_limit_in_bytes(); ++ ++ // consistent with other container engines, ++ // not obtained other memory info ++ ++ return true; ++} ++ ++static bool NRIHugePageLimitFromCRI(const runtime::v1::LinuxContainerResources &config, nri_linux_resources &resources) ++{ ++ int i; ++ nri_hugepage_limit *tmp = nullptr; ++ ++ if (config.hugepage_limits_size() == 0) { ++ return true; ++ } ++ ++ resources.hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ config.hugepage_limits_size()); ++ if (resources.hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < config.hugepage_limits_size(); i++) { ++ tmp = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (tmp == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ tmp->page_size = util_strdup_s(config.hugepage_limits(i).page_size().c_str()); ++ tmp->limit = config.hugepage_limits(i).limit(); ++ resources.hugepage_limits[i] = tmp; ++ resources.hugepage_limits_len++; ++ tmp = nullptr; ++ } ++ return true; ++} ++ ++static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources &config, ++ nri_linux_resources &resources) -> bool ++{ ++ if (!NRILinuxMemoryFromCRI(config, *resources.memory)) { ++ ERROR("Failed to transform memory to nri for container"); ++ return false; ++ } ++ ++ if (!NRILinuxCpuFromCRI(config, *resources.cpu)) { ++ ERROR("Failed to transform cpu to nri for container"); ++ return false; ++ } ++ ++ if (!NRIHugePageLimitFromCRI(config, resources)) { ++ ERROR("Failed to transform hugepage limits to nri for container"); ++ return false; ++ } ++ ++ // resources.blockio_class is not support ++ // resources.rdt_class is not support ++ // They are not standard fields in oci spec ++ ++ Errors tmpError; ++ ++ resources.unified = Transform::ProtobufMapToJsonMapForString(config.unified(), tmpError); ++ if (resources.unified == nullptr) { ++ ERROR("Failed to transform unified to nri for container : %s", tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ // resources.devices is not set in pod ++ ++ return true; ++} ++ ++static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool ++{ ++ if (!init_nri_linux_resources(&linux.pod_overhead)) { ++ ERROR("Failed to init nri linux overhead resources for pod"); ++ return false; ++ } ++ if (!init_nri_linux_resources(&linux.pod_resources)) { ++ ERROR("Failed to init nri linux resources resources for pod"); ++ return false; ++ } ++ if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { ++ ERROR("Failed to transform overhead to nri for pod"); ++ return false; ++ } ++ ++ if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { ++ ERROR("Failed to transform resources to nri for pod"); ++ return false; ++ } ++ ++ linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); ++ ++ // todo: other container engines get linux.cgroups_path/linux.resourses/linux.namespace from spec.linux, ++ // How does isulad get these values ​​from CRI module? ++ return true; ++} ++ ++auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool ++{ ++ container_t *cont = nullptr; ++ Errors tmpError; ++ ++ cont = containers_store_get(sandbox->GetName().c_str()); ++ if (cont != nullptr) { ++ pod.pid = container_state_get_pid(cont->state); ++ container_unref(cont); ++ } ++ ++ pod.id = util_strdup_s(sandbox->GetId().c_str()); ++ pod.name = util_strdup_s(sandbox->GetName().c_str()); ++ if (sandbox->GetSandboxConfig().has_metadata()) { ++ pod.uid = util_strdup_s(sandbox->GetSandboxConfig().metadata().uid().c_str()); ++ pod._namespace = util_strdup_s(sandbox->GetSandboxConfig().metadata().namespace_().c_str()); ++ } ++ ++ ++ pod.labels = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().labels(), tmpError); ++ if (pod.labels == nullptr) { ++ ERROR("Failed to transform labels to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ pod.annotations = Transform::ProtobufMapToJsonMapForString(sandbox->GetSandboxConfig().annotations(), tmpError); ++ if (pod.annotations == nullptr) { ++ ERROR("Failed to transform annotations to nri for pod : %s, : %s", pod.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ if (sandbox->GetSandboxConfig().has_linux()) { ++ pod.linux = (nri_linux_pod_sandbox *)util_common_calloc_s(sizeof(nri_linux_pod_sandbox)); ++ if (pod.linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!NRILinuxFromCRI(sandbox->GetSandboxConfig().linux(), *pod.linux)) { ++ ERROR("Failed to transform linux to nri for pod : %s", pod.name); ++ return false; ++ } ++ } ++ ++ pod.runtime_handler = util_strdup_s(sandbox->GetRuntimeHandle().c_str()); ++ ++ return true; ++} ++ ++static auto CRIMountArrToNRI(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool ++{ ++ size_t i, len; ++ ++ // get mount from cont ++ len = containerConfig.mounts_size(); ++ if (len == 0) { ++ return true; ++ } ++ con.mounts = (nri_mount **)util_smart_calloc_s(sizeof(nri_mount *), len); ++ if (con.mounts == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ nri_mount *tmp = nullptr; ++ ++ for (i = 0; i < len; i++) { ++ tmp = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ if (tmp == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ if (containerConfig.mounts()[i].container_path().empty() || containerConfig.mounts()[i].host_path().empty()) { ++ ERROR("Mount path is empty"); ++ goto error_out; ++ } ++ ++ char path[PATH_MAX] = { 0 }; ++ if (!util_clean_path(containerConfig.mounts()[i].container_path().c_str(), path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", containerConfig.mounts()[i].container_path().c_str()); ++ goto error_out; ++ } ++ ++ tmp->destination = util_strdup_s(path); ++ ++ if (!util_clean_path(containerConfig.mounts()[i].host_path().c_str(), path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", containerConfig.mounts()[i].host_path().c_str()); ++ goto error_out; ++ } ++ tmp->source = util_strdup_s(path); ++ ++ if (util_array_append(&(tmp->options), "rbind") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ ++ if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_PRIVATE) { ++ DEBUG("noop, private is default"); ++ if (util_array_append(&(tmp->options), "rprivate") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_BIDIRECTIONAL) { ++ if (util_array_append(&(tmp->options), "rshared") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else if (containerConfig.mounts()[i].propagation() == runtime::v1::PROPAGATION_HOST_TO_CONTAINER) { ++ if (util_array_append(&(tmp->options), "rslave") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ WARN("unknown propagation mode for hostPath %s", containerConfig.mounts()[i].host_path().c_str()); ++ if (util_array_append(&(tmp->options), "rprivate") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ if (containerConfig.mounts()[i].readonly()) { ++ if (util_array_append(&(tmp->options), "ro") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ if (util_array_append(&(tmp->options), "rw") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ tmp->type = util_strdup_s("bind"); ++ ++ con.mounts[i] = tmp; ++ tmp = nullptr; ++ con.mounts_len++; ++ } ++ return true; ++ ++error_out: ++ free_nri_mount(tmp); ++ return false; ++} ++ ++static auto MountPointsElementToNRI(container_config_v2_common_config_mount_points *mp, nri_container &con) -> bool ++{ ++ size_t i, len; ++ nri_mount *tmp = nullptr; ++ ++ if (mp == nullptr || mp->len == 0) { ++ return true; ++ } ++ len = mp->len; ++ ++ con.mounts = (nri_mount **)util_smart_calloc_s(sizeof(nri_mount *), len); ++ if (con.mounts == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < len; i++) { ++ tmp = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ char path[PATH_MAX] = { 0 }; ++ ++ if (!util_clean_path(mp->values[i]->destination, path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount dest path: %s", mp->values[i]->destination); ++ goto error_out; ++ } ++ tmp->destination = util_strdup_s(path); ++ ++ if (!util_clean_path(mp->values[i]->source, path, sizeof(path))) { ++ ERROR("Failed to get clean path for mount src path: %s", mp->values[i]->source); ++ goto error_out; ++ } ++ tmp->source = util_strdup_s(path); ++ ++ if (util_array_append(&(tmp->options), "rbind") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ if (util_array_append(&(tmp->options), mp->values[i]->propagation) != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ ++ if (mp->values[i]->rw) { ++ if (util_array_append(&(tmp->options), "rw") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } else { ++ if (util_array_append(&(tmp->options), "ro") != 0) { ++ ERROR("Failed to append options"); ++ goto error_out; ++ } ++ } ++ ++ tmp->type = util_strdup_s("bind"); ++ con.mounts[i] = tmp; ++ con.mounts_len++; ++ tmp = nullptr; ++ } ++ ++ return true; ++ ++error_out: ++ free_nri_mount(tmp); ++ return false; ++} ++ ++// container info is incomplete because container in excution is not created ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool ++{ ++ // todo: can not get container id and state from containerConfig ++ if (containerConfig.has_metadata() && !containerConfig.metadata().name().empty()) { ++ con.name = util_strdup_s(containerConfig.metadata().name().c_str()); ++ } ++ ++ Errors tmpError; ++ ++ con.labels = Transform::ProtobufMapToJsonMapForString(containerConfig.labels(), tmpError); ++ if (con.labels == nullptr) { ++ ERROR("Failed to transform labels to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ con.annotations = Transform::ProtobufMapToJsonMapForString(containerConfig.annotations(), tmpError); ++ if (con.annotations == nullptr) { ++ ERROR("Failed to transform annotations to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ ++ con.args = Transform::RepeatedPtrFieldToCharArray(containerConfig.args()); ++ if (con.args == nullptr) { ++ ERROR("Failed to transform args to nri for con : %s, : %s", con.name, tmpError.GetMessage().c_str()); ++ return false; ++ } ++ con.args_len = containerConfig.args_size(); ++ ++ auto envVect = CRIHelpersV1::GenerateEnvList(containerConfig.envs()); ++ con.env = Transform::StringVectorToCharArray(envVect); ++ if (con.env == nullptr) { ++ ERROR("Failed to transform env to nri for con : %s", con.name); ++ return false; ++ } ++ con.env_len = containerConfig.envs_size(); ++ ++ if (!CRIMountArrToNRI(containerConfig, con)) { ++ ERROR("Failed to transform mounts to nri for con : %s", con.name); ++ return false; ++ } ++ return true; ++ ++ // todo: can not get container hooks and pid from containerConfig ++} ++ ++// container info is incomplete because container in excution is not created ++auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool ++{ ++ container_t *cont = nullptr; ++ bool ret = false; ++ ++ cont = containers_store_get(id.c_str()); ++ if (cont == nullptr || cont->common_config == nullptr) { ++ ERROR("No such container:%s", id.c_str()); ++ goto out; ++ } ++ ++ con.id = util_strdup_s(id.c_str()); ++ ++ con.name = util_strdup_s(cont->common_config->name); ++ ++ con.labels = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (con.labels == nullptr) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ con.annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (con.annotations == nullptr) { ++ ERROR("Out of memory"); ++ goto out; ++ } ++ // state ++ if (dup_json_map_string_string(cont->common_config->config->labels, con.labels) != 0) { ++ ERROR("Failed to copy labels for con: %s", cont->common_config->name); ++ goto out; ++ } ++ if (dup_json_map_string_string(cont->common_config->config->annotations, con.annotations) != 0) { ++ ERROR("Failed to copy labels for con: %s", cont->common_config->name); ++ goto out; ++ } ++ ++ con.args = util_copy_array_by_len(cont->common_config->args, cont->common_config->args_len); ++ if (cont->common_config->args_len != 0 && con.args == nullptr) { ++ ERROR("Failed to copy args for con: %s", cont->common_config->name); ++ goto out; ++ } ++ con.args_len = cont->common_config->args_len; ++ ++ con.env = util_copy_array_by_len(cont->common_config->config->env, cont->common_config->config->env_len); ++ if (cont->common_config->config->env_len != 0 && con.env == nullptr) { ++ ERROR("Failed to copy env for con: %s", cont->common_config->name); ++ goto out; ++ } ++ con.env_len = cont->common_config->config->env_len; ++ ++ if (!MountPointsElementToNRI(cont->common_config->mount_points, con)) { ++ ERROR("Failed to transform mounts to nri for con : %s", con.name); ++ goto out; ++ } ++ ++ // todo: can convert hostconfig's hook_spec to nri spec ++ ++ con.pid = container_state_get_pid(cont->state); ++ if (con.pid < 0) { ++ ERROR("Container %s pid %d invalid", cont->common_config->name, con.pid); ++ goto out; ++ } ++ ++ con.pod_sandbox_id = util_strdup_s(cont->common_config->sandbox_info->id); ++ ret = true; ++ ++out: ++ container_unref(cont); ++ return ret; ++} ++ ++auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool ++{ ++ if (src == nullptr) { ++ return false; ++ } ++ ++ if (src->memory != nullptr) { ++ resources.set_memory_limit_in_bytes(*src->memory->limit); ++ resources.set_oom_score_adj(DefaultOOMScoreAdj); ++ } ++ ++ if (src->cpu != nullptr) { ++ if (src->cpu->shares != NULL) { ++ resources.set_cpu_shares(*src->cpu->shares); ++ } ++ if (src->cpu->quota != NULL) { ++ resources.set_cpu_quota(*src->cpu->quota); ++ } ++ if (src->cpu->period != NULL) { ++ resources.set_cpu_period(*src->cpu->period); ++ } ++ ++ resources.set_cpuset_cpus(src->cpu->cpus); ++ resources.set_cpuset_mems(src->cpu->mems); ++ } ++ ++ if (src->hugepage_limits != nullptr && src->hugepage_limits_len > 0) { ++ for (size_t i = 0; i < src->hugepage_limits_len; i++) { ++ if (src->hugepage_limits[i] != nullptr) { ++ auto limit = resources.add_hugepage_limits(); ++ limit->set_page_size(src->hugepage_limits[i]->page_size); ++ limit->set_limit(src->hugepage_limits[i]->limit); ++ } ++ } ++ } ++ ++ if (src->unified != nullptr) { ++ Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); ++ } ++ ++ return true; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +index 883f7c41..c04b14e4 100644 +--- a/src/daemon/common/nri/nri_convert.h ++++ b/src/daemon/common/nri/nri_convert.h +@@ -27,10 +27,11 @@ + #include "sandbox.h" + #include "api_v1.pb.h" + +-auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox *pod) -> bool; +-auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container *con) -> bool; +-auto ContainerToNRIByID(const std::string &id, nri_container *con) -> bool; +-auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, int pod_len) -> bool; ++auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool; ++auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; ++auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; ++auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, ++ int pod_len) -> bool; + + auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; + #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c +new file mode 100644 +index 00000000..855fe3b3 +--- /dev/null ++++ b/src/daemon/common/nri/nri_spec.c +@@ -0,0 +1,602 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri oci functions ++ *********************************************************************************/ ++ ++#include "nri_spec.h" ++ ++#include ++ ++#include "map.h" ++#include "utils.h" ++#include "utils_string.h" ++#include "nri_utils.h" ++#include "specs_api.h" ++#include "sysinfo.h" ++#include "verify.h" ++#include "specs_extend.h" ++ ++static defs_hook *nri_hook_to_oci(const nri_hook *h) ++{ ++ defs_hook *oci_hook = NULL; ++ ++ if (h == NULL) { ++ return NULL; ++ } ++ ++ oci_hook = util_common_calloc_s(sizeof(*oci_hook)); ++ if (oci_hook == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_hook->path = util_strdup_s(h->path); ++ if (h->args_len != 0) { ++ oci_hook->args = util_copy_array_by_len(h->args, h->args_len); ++ if (oci_hook->args == NULL) { ++ ERROR("Failed to copy args"); ++ goto error_out; ++ } ++ oci_hook->args_len = h->args_len; ++ } ++ if (h->env_len != 0) { ++ oci_hook->env = util_copy_array_by_len(h->env, h->env_len); ++ if (oci_hook->env == NULL) { ++ ERROR("Failed to copy env"); ++ goto error_out; ++ } ++ oci_hook->env_len = h->env_len; ++ } ++ if (h->timeout != NULL) { ++ oci_hook->timeout = *(h->timeout); ++ } ++ return oci_hook; ++ ++error_out: ++ free_defs_hook(oci_hook); ++ return NULL; ++} ++ ++static defs_device *nri_device_to_oci(nri_linux_device *dev) ++{ ++ if (dev == NULL) { ++ return NULL; ++ } ++ ++ defs_device *oci_dev = util_common_calloc_s(sizeof(defs_device)); ++ if (oci_dev == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_dev->path = util_strdup_s(dev->path); ++ oci_dev->type = util_strdup_s(dev->type); ++ oci_dev->major = dev->major; ++ oci_dev->minor = dev->minor; ++ if (dev->file_mode != NULL) { ++ oci_dev->file_mode = *dev->file_mode; ++ } ++ if (dev->uid != NULL) { ++ oci_dev->uid = *dev->uid; ++ } ++ if (dev->gid != NULL) { ++ oci_dev->gid = *dev->gid; ++ } ++ ++ return oci_dev; ++} ++ ++static defs_mount *nri_mount_to_oci(nri_mount *mount) ++{ ++ if (mount == NULL) { ++ return NULL; ++ } ++ ++ defs_mount *oci_mount = util_common_calloc_s(sizeof(defs_mount)); ++ if (oci_mount == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ oci_mount->destination = util_strdup_s(mount->destination); ++ oci_mount->type = util_strdup_s(mount->type); ++ oci_mount->source = util_strdup_s(mount->source); ++ if (mount->options_len != 0) { ++ oci_mount->options = util_copy_array_by_len(mount->options, mount->options_len); ++ if (oci_mount->options == NULL) { ++ ERROR("Failed to copy options"); ++ free_defs_mount(oci_mount); ++ return NULL; ++ } ++ oci_mount->options_len = mount->options_len; ++ } ++ ++ return oci_mount; ++} ++ ++static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ int ret = -1; ++ size_t i; ++ ++ if (adjust == NULL || adjust->annotations == NULL || adjust->annotations->len == 0) { ++ return 0; ++ } ++ ++ if (make_sure_oci_spec_annotations(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec annotations"); ++ return -1; ++ } ++ ++ json_map_string_string *cleard = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (cleard == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ map_t *del = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ if (del == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < adjust->annotations->len; i++) { ++ __isula_auto_free char *out = NULL; ++ if (is_marked_for_removal(adjust->annotations->keys[i], &out)) { ++ if (!map_insert(del, out, "")) { ++ ERROR("Failed to insert del map"); ++ goto free_out; ++ } ++ continue; ++ } ++ if (append_json_map_string_string(cleard, adjust->annotations->keys[i], ++ adjust->annotations->values[i]) != 0) { ++ ERROR("Failed to append annotation"); ++ goto free_out; ++ } ++ } ++ ++ for (i = 0; i < oci_spec->annotations->len; i++) { ++ if (map_search(del, oci_spec->annotations->keys[i]) != NULL) { ++ continue; ++ } ++ append_json_map_string_string(cleard, oci_spec->annotations->keys[i], ++ oci_spec->annotations->values[i]); ++ } ++ ++ free_json_map_string_string(oci_spec->annotations); ++ oci_spec->annotations = cleard; ++ ret = 0; ++ ++free_out: ++ free_json_map_string_string(cleard); ++ map_free(del); ++ return ret; ++} ++ ++static void nri_key_value_map_kvfree(void *key, void *value) ++{ ++ free(key); ++ ++ // no need to free nri_key_value ++ // nri_key_value *value will be free in nri_container_adjustment *adjust ++} ++ ++ ++static int nri_adjust_env(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ int ret = -1; ++ size_t i; ++ char **old_env = NULL; ++ size_t old_env_len = 0; ++ __isula_auto_array_t char **adjust_env = NULL; ++ size_t adjust_env_len = 0; ++ ++ if (adjust->env == NULL || adjust->env_len == 0) { ++ return 0; ++ } ++ ++ map_t *mod = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, nri_key_value_map_kvfree); ++ if (mod == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < adjust->env_len; i++) { ++ nri_key_value *e = adjust->env[i]; ++ char *out = NULL; ++ (void)is_marked_for_removal(e->key, &out); ++ ++ if (!map_insert(mod, out, e) == false) { ++ ERROR("Failed to insert mod map"); ++ goto free_out; ++ } ++ } ++ ++ if (map_size(mod) <= 0 || oci_spec == NULL || oci_spec->process == NULL) { ++ ret = 0; ++ goto free_out; ++ } ++ ++ // modify existing environment ++ old_env = oci_spec->process->env; ++ old_env_len = oci_spec->process->env_len; ++ oci_spec->process->env = NULL; ++ oci_spec->process->env_len = 0; ++ ++ for (i = 0; i < old_env_len; i++) { ++ __isula_auto_array_t char **envArr = util_string_split_n(old_env[i], '=', 2); ++ if (envArr == NULL) { ++ continue; ++ } ++ ++ nri_key_value *target = map_search(mod, envArr[0]); ++ if (target != NULL) { ++ __isula_auto_free char *out = NULL; ++ if (!is_marked_for_removal(envArr[0], &out)) { ++ // If not marked for removal, append modified value ++ __isula_auto_free char *tmp_str = util_string_append(target->key, "="); ++ __isula_auto_free char *final_str = util_string_append(tmp_str, target->value); ++ ++ if (util_array_append(&adjust_env, final_str) != 0) { ++ ERROR("Failed to append env"); ++ goto free_out; ++ } ++ adjust_env_len++; ++ continue; ++ } ++ } ++ // If not found in mod map, append original value ++ if (util_array_append(&adjust_env, old_env[i]) != 0) { ++ ERROR("Failed to append env"); ++ goto free_out; ++ } ++ adjust_env_len++; ++ } ++ ++ ret = 0; ++free_out: ++ if (merge_env(oci_spec, (const char **)adjust_env, adjust_env_len) != 0) { ++ ERROR("Failed to merge env"); ++ goto free_out; ++ } ++ for (i = 0; i < old_env_len; i++) { ++ free(old_env[i]); ++ } ++ free(old_env); ++ map_free(mod); ++ return ret; ++} ++ ++static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->hooks == NULL) { ++ return 0; ++ } ++ ++ size_t i; ++ int ret = 0; ++ ++ if (make_sure_oci_spec_hooks(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec hooks"); ++ return -1; ++ } ++ ++ // todo: change to macro definition function call ++ for (i = 0; i < adjust->hooks->prestart_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->prestart[i]); ++ ret = spec_add_prestart_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->prestart[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ ++ for (i = 0; i < adjust->hooks->poststart_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->poststart[i]); ++ ret = spec_add_poststart_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->poststart[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ ++ for (i = 0; i < adjust->hooks->poststop_len; i++) { ++ defs_hook *oci_hook = nri_hook_to_oci(adjust->hooks->poststop[i]); ++ ret = spec_add_poststop_hook(oci_spec, oci_hook); ++ if (ret != 0) { ++ ERROR("Failed add hook %s", adjust->hooks->poststop[i]->path); ++ free_defs_hook(oci_hook); ++ return -1; ++ } ++ } ++ /* ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ ++ return ret; ++} ++ ++static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < adjust->linux->devices_len; i++) { ++ nri_linux_device *dev = adjust->linux->devices[i]; ++ if (spec_add_device(oci_spec, nri_device_to_oci(dev)) != 0) { ++ ERROR("Failed to add device %s", dev->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { ++ return 0; ++ } ++ ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = util_strdup_s(adjust->linux->cgroups_path); ++ ++ return 0; ++} ++ ++static void nri_adjust_cpu_memory(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ if (resource->cpu == NULL) { ++ return; ++ } ++ if (make_sure_oci_spec_linux_resources_cpu(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec linux resources cpu"); ++ return; ++ } ++ if (resource->cpu->shares != NULL) { ++ oci_spec->linux->resources->cpu->shares = *resource->cpu->shares; ++ } ++ if (resource->cpu->quota != NULL) { ++ oci_spec->linux->resources->cpu->quota = *resource->cpu->quota; ++ } ++ if (resource->cpu->period != NULL) { ++ oci_spec->linux->resources->cpu->period = *resource->cpu->period; ++ } ++ if (resource->cpu->realtime_runtime != NULL) { ++ oci_spec->linux->resources->cpu->realtime_runtime = *resource->cpu->realtime_runtime; ++ } ++ if (resource->cpu->realtime_period != NULL) { ++ oci_spec->linux->resources->cpu->realtime_period = *resource->cpu->realtime_period; ++ } ++} ++ ++static void nri_adjust_memory_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ if (resource->memory == NULL) { ++ return; ++ } ++ ++ if (make_sure_oci_spec_linux_resources_mem(oci_spec) != 0) { ++ ERROR("Failed to make sure oci spec linux resources memory"); ++ return; ++ } ++ if (resource->memory->limit != NULL) { ++ oci_spec->linux->resources->memory->limit = *resource->memory->limit; ++ } ++ if (resource->memory->reservation != NULL) { ++ oci_spec->linux->resources->memory->reservation = *resource->memory->reservation; ++ } ++ if (resource->memory->swap != NULL) { ++ oci_spec->linux->resources->memory->swap = *resource->memory->swap; ++ } ++ if (resource->memory->kernel != NULL) { ++ oci_spec->linux->resources->memory->kernel = *resource->memory->kernel; ++ } ++ if (resource->memory->kernel_tcp != NULL) { ++ oci_spec->linux->resources->memory->kernel_tcp = *resource->memory->kernel_tcp; ++ } ++ if (resource->memory->swappiness != NULL) { ++ oci_spec->linux->resources->memory->swappiness = *resource->memory->swappiness; ++ } ++ if (resource->memory->disable_oom_killer != NULL) { ++ oci_spec->linux->resources->memory->disable_oom_killer = *resource->memory->disable_oom_killer; ++ } ++} ++ ++static int nri_adjust_hugepage_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ size_t i; ++ if (resource->hugepage_limits != NULL) { ++ for (i = 0; i < resource->hugepage_limits_len; i++) { ++ nri_hugepage_limit *limit = resource->hugepage_limits[i]; ++ if (limit->page_size != NULL) { ++ if (spec_add_linux_resources_hugepage_limit(oci_spec, limit->page_size, limit->limit) != 0) { ++ ERROR("Failed to add hugepage limit"); ++ return -1; ++ } ++ } ++ } ++ } ++ return 0; ++} ++ ++static int nri_adjust_unified_resource(nri_linux_resources *resource, oci_runtime_spec *oci_spec) ++{ ++ size_t i; ++ if (resource->unified != NULL) { ++ for (i = 0; i < resource->unified->len; i++) { ++ if (append_json_map_string_string(oci_spec->linux->resources->unified, resource->unified->keys[i], ++ resource->unified->values[i]) != 0) { ++ ERROR("Failed to append unified resource"); ++ return -1; ++ } ++ } ++ } ++ return 0; ++} ++ ++static int nri_adjust_resources(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->resources == NULL) { ++ return 0; ++ } ++ ++ nri_linux_resources *resource = adjust->linux->resources; ++ ++ nri_adjust_memory_resource(resource, oci_spec); ++ nri_adjust_cpu_memory(resource, oci_spec); ++ ++ if (nri_adjust_hugepage_resource(resource, oci_spec) != 0) { ++ ERROR("Failed to adjust hugepage resource"); ++ return -1; ++ } ++ ++ if (nri_adjust_unified_resource(resource, oci_spec) != 0) { ++ ERROR("Failed to adjust unified resource"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->mounts == NULL || adjust->mounts_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->mounts_len; i++) { ++ nri_mount *mount = adjust->mounts[i]; ++ defs_mount *oci_mount = nri_mount_to_oci(mount); ++ if (oci_mount == NULL) { ++ ERROR("Failed to convert nri mount to oci mount"); ++ return -1; ++ } ++ if (spec_add_mount(oci_spec, oci_mount) != 0) { ++ ERROR("Failed to add mount"); ++ free_defs_mount(oci_mount); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->rlimits_len; i++) { ++ nri_posix_rlimit *rlimit = adjust->rlimits[i]; ++ if (rlimit->type == NULL) { ++ ERROR("Invalid rlimit type"); ++ return -1; ++ } ++ if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { ++ ERROR("Failed to add rlimit"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++// todo: we do not support it blockio_class ++static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->resources->blockio_class == NULL) { ++ return 0; ++ } ++ ++ return 0; ++} ++ ++int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++{ ++ if (oci_spec == NULL || adjust == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (nri_adjust_annotation(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust annotation in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_env(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust env in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_hooks(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust hooks in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_devices(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust devices in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_cgroup_path(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust cgroup path in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_resources(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust resources in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_blockio_class(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust blockio class in oci spec"); ++ return -1; ++ } ++ ++ // iSuald is not support IntelRdt ++ if (nri_adjust_mounts(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust mount in oci spec"); ++ return -1; ++ } ++ ++ if (nri_adjust_rlimit(adjust, oci_spec) != 0) { ++ ERROR("Failed to do nri adjust rlimit in oci spec"); ++ return -1; ++ } ++ ++ __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Failed to get system info"); ++ return -1; ++ } ++ ++ if (verify_container_settings(oci_spec, sysinfo) != 0) { ++ ERROR("Failed to verify oci runtime spec settings after adjust by nri"); ++ return -1; ++ } ++ ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c +new file mode 100644 +index 00000000..51054e32 +--- /dev/null ++++ b/src/daemon/common/nri/nri_utils.c +@@ -0,0 +1,520 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-17 ++ * Description: provide nri utils functions ++ *********************************************************************************/ ++ ++#include "nri_utils.h" ++ ++#include ++ ++#include "utils.h" ++ ++static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (*dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*dest)->limit = src->limit; ++ (*dest)->page_size = util_strdup_s(src->page_size); ++ return true; ++} ++ ++static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->args = util_copy_array_by_len(src->args, src->args_len); ++ (*dest)->args_len = src->args_len; ++ (*dest)->env = util_copy_array_by_len(src->env, src->env_len); ++ (*dest)->env_len = src->env_len; ++ (*dest)->path = util_strdup_s(src->path); ++ return true; ++} ++ ++static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->allow = src->allow; ++ (*dest)->type = util_strdup_s(src->type); ++ (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->major == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->minor == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->access = util_strdup_s(src->access); ++ return true; ++} ++ ++static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if ((*dest) == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->cpus = util_strdup_s(src->cpus); ++ (*dest)->mems = util_strdup_s(src->mems); ++ if (src->period != NULL) { ++ (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->period == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->period = *src->period; ++ } ++ ++ if (src->quota != NULL) { ++ (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->quota == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->quota = *src->quota; ++ } ++ ++ if (src->realtime_period != NULL) { ++ (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->realtime_period == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->realtime_period = *src->realtime_period; ++ } ++ ++ if (src->realtime_runtime != NULL) { ++ (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->realtime_runtime == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->realtime_runtime = *src->realtime_runtime; ++ } ++ ++ if (src->shares != NULL) { ++ (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->shares == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->shares = *src->shares; ++ } ++ ++ return true; ++} ++ ++static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (src->limit != NULL) { ++ (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->limit == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->limit = *src->limit; ++ } ++ ++ if (src->reservation != NULL) { ++ (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->reservation == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->reservation = *src->reservation; ++ } ++ ++ if (src->swap != NULL) { ++ (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->swap == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->swap = *src->swap; ++ } ++ ++ if (src->kernel != NULL) { ++ (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->kernel == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->kernel = *src->kernel; ++ } ++ ++ ++ if (src->kernel_tcp != NULL) { ++ (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if ((*dest)->kernel_tcp == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->kernel_tcp = *src->kernel_tcp; ++ } ++ ++ if (src->swappiness != NULL) { ++ (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if ((*dest)->swappiness == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->swappiness = *src->swappiness; ++ } ++ ++ if (src->disable_oom_killer != NULL) { ++ (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if ((*dest)->disable_oom_killer == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->disable_oom_killer = *src->disable_oom_killer; ++ } ++ ++ if (src->use_hierarchy != NULL) { ++ (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if ((*dest)->use_hierarchy == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ *(*dest)->use_hierarchy = *src->use_hierarchy; ++ } ++ return true; ++} ++ ++bool is_marked_for_removal(const char* key, char **out) ++{ ++ if (key == NULL || out == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ if (!util_has_prefix(key, "-")) { ++ *out = (char*)key; ++ return false; ++ } ++ ++ *out = util_sub_string(key, 1, strlen(key) - 1); ++ if (*out == NULL) { ++ ERROR("Failed to sub string"); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool copy_nri_mount(const nri_mount *src, nri_mount **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->destination = util_strdup_s(src->destination); ++ (*dest)->options = util_copy_array_by_len(src->options, src->options_len); ++ (*dest)->options_len = src->options_len; ++ (*dest)->source = util_strdup_s(src->source); ++ (*dest)->type = util_strdup_s(src->type); ++ return true; ++} ++ ++bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->key = util_strdup_s(src->key); ++ (*dest)->value = util_strdup_s(src->value); ++ return true; ++} ++ ++bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ (*dest)->hard = src->hard; ++ (*dest)->soft = src->soft; ++ (*dest)->type = util_strdup_s(src->type); ++ return true; ++} ++ ++bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) ++{ ++ if (src == NULL || dest == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (*dest == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (!init_nri_linux_resources(dest)) { ++ ERROR("Failed to init dest nri linux resources"); ++ goto free_out; ++ } ++ ++ if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { ++ ERROR("Failed to copy nri_linux_cpu"); ++ goto free_out; ++ } ++ ++ if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { ++ ERROR("Failed to copy nri_linux_memory"); ++ goto free_out; ++ } ++ ++ (*dest)->blockio_class = util_strdup_s(src->blockio_class); ++ (*dest)->rdt_class = util_strdup_s(src->rdt_class); ++ ++ if (src->hugepage_limits_len > 0) { ++ (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), ++ src->hugepage_limits_len); ++ for (size_t i = 0; i < src->hugepage_limits_len; ++i) { ++ if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { ++ ERROR("Failed to copy nri_hugepage_limit"); ++ goto free_out; ++ } ++ } ++ } ++ ++ if (src->devices_len > 0) { ++ (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); ++ for (size_t i = 0; i < src->devices_len; ++i) { ++ if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { ++ ERROR("Failed to copy nri_linux_device_cgroup"); ++ goto free_out; ++ } ++ } ++ } ++ ++ if (dup_json_map_string_string(src->unified, (*dest)->unified)) { ++ ERROR("Failed to copy json_map_string_string"); ++ goto free_out; ++ } ++ ++ return true; ++ ++free_out: ++ free_nri_linux_resources(*dest); ++ return false; ++} ++ ++bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, ++ size_t sourceLen) ++{ ++ size_t oldSize = targetSize * sizeof(nri_hook *); ++ size_t newSize = oldSize + sourceLen * sizeof(nri_hook *); ++ ++ if (sourceHooks == NULL || targetHooks == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { ++ ERROR("Failed to realloc and assign hook array"); ++ return false; ++ } ++ ++ for (size_t i = 0; i < sourceLen; i++) { ++ if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { ++ ERROR("Failed to copy hook"); ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++bool init_nri_container_adjust(nri_container_adjustment **adjust) ++{ ++ if (adjust == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); ++ if (*adjust == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if ((*adjust)->annotations == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); ++ if ((*adjust)->env == NULL) { ++ goto free_out; ++ } ++ (*adjust)->env_len = 0; ++ ++ (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); ++ if ((*adjust)->hooks == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); ++ if ((*adjust)->linux == NULL) { ++ goto free_out; ++ } ++ ++ (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if ((*adjust)->linux->resources == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); ++ if ((*adjust)->mounts == NULL) { ++ goto free_out; ++ } ++ (*adjust)->mounts_len = 0; ++ ++ (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); ++ if ((*adjust)->rlimits == NULL) { ++ goto free_out; ++ } ++ (*adjust)->rlimits_len = 0; ++ ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_container_adjustment(*adjust); ++ return false; ++} ++ ++bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) ++{ ++ if (update == NULL || id == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); ++ if (*update == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*update)->container_id = util_strdup_s(id); ++ (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); ++ if ((*update)->linux == NULL) { ++ goto free_out; ++ } ++ ++ (*update)->ignore_failure = ignore_failure; ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_container_update(*update); ++ return false; ++} ++ ++bool init_nri_linux_resources(nri_linux_resources **resources) ++{ ++ if (resources == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; ++ } ++ ++ *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (*resources == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if ((*resources)->cpu == NULL) { ++ goto free_out; ++ } ++ ++ (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if ((*resources)->memory == NULL) { ++ goto free_out; ++ } ++ ++ (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if ((*resources)->unified == NULL) { ++ goto free_out; ++ } ++ return true; ++ ++free_out: ++ ERROR("Out of memory"); ++ free_nri_linux_resources(*resources); ++ return false; ++} +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +index 3aa50ae4..7bf54a71 100644 +--- a/src/daemon/common/nri/nri_utils.h ++++ b/src/daemon/common/nri/nri_utils.h +@@ -51,14 +51,13 @@ typedef enum { + + bool copy_nri_mount(const nri_mount *src, nri_mount **dest); + bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +-bool copy_nri_hook(const nri_hook *src, nri_hook **dest); + bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); + bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); +-bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest); + + bool is_marked_for_removal(const char* key, char **out); + +-bool realloc_and_copy_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, size_t sourceLen); ++bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, ++ size_t sourceLen); + + bool init_nri_container_adjust(nri_container_adjustment **adjust); + bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index d7b54498..9ba1c8a0 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -456,6 +456,175 @@ out: + (void)isulad_server_conf_unlock(); + return path; + } ++ ++#ifdef ENABLE_NRI ++bool conf_get_nri_support(void) ++{ ++ bool nri_support = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ nri_support = conf->json_confs->nri_support; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return nri_support; ++} ++ ++bool conf_get_nri_external_support(void) ++{ ++ bool nri_external_support = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ nri_external_support = conf->json_confs->disable_connections; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return !nri_external_support; ++} ++ ++char *conf_get_nri_plugin_config_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL || conf->json_confs->plugin_config_path == NULL) { ++ path = util_strdup_s(DEFAULT_PLUGIN_CONFIG_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->plugin_config_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++char *conf_get_nri_plugin_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_path == NULL) { ++ path = util_strdup_s(DEFAULT_PLUGIN_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->plugin_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++char *conf_get_socket_path(void) ++{ ++ char *path = NULL; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return NULL; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->nri_socket_path == NULL) { ++ path = util_strdup_s(DEFAULT_SOCKET_PATH); ++ goto out; ++ } ++ ++ path = util_strdup_s(conf->json_confs->nri_socket_path); ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return path; ++} ++ ++uint64_t conf_get_nri_plugin_registration_timeout(void) ++{ ++ uint64_t timeout = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_registration_timeout == 0) { ++ timeout = DEFAULT_PLUGIN_REGISTRY_TIMEOUT; ++ goto out; ++ } ++ ++ timeout = conf->json_confs->plugin_registration_timeout; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return timeout; ++} ++uint64_t conf_get_nri_plugin_requst_timeout(void) ++{ ++ uint64_t timeout = false; ++ struct service_arguments *conf = NULL; ++ ++ if (isulad_server_conf_rdlock() != 0) { ++ return false; ++ } ++ ++ conf = conf_get_server_conf(); ++ if (conf == NULL || conf->json_confs == NULL) { ++ goto out; ++ } ++ ++ if (conf->json_confs->plugin_requst_timeout == 0) { ++ timeout = DEFAULT_PLUGIN_REQUST_TIMEOUT; ++ goto out; ++ } ++ ++ timeout = conf->json_confs->plugin_requst_timeout; ++ ++out: ++ (void)isulad_server_conf_unlock(); ++ return timeout; ++} ++#endif + #endif + + /* conf get isulad rootdir */ +@@ -1762,6 +1931,15 @@ int merge_json_confs_into_global(struct service_arguments *args) + tmp_json_confs->cri_sandboxers = NULL; + #endif + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; ++#ifdef ENABLE_NRI ++ args->json_confs->nri_support = tmp_json_confs->nri_support; ++ args->json_confs->disable_connections = tmp_json_confs->disable_connections; ++ override_string_value(&args->json_confs->plugin_config_path, &tmp_json_confs->plugin_config_path); ++ override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); ++ args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; ++ args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; ++ override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); ++#endif + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif + +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 6a1cd776..d5ea0c7c 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -76,6 +76,11 @@ int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const ch + int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type, uint64_t hard, uint64_t soft); + #endif /* ENABLE_NRI */ + ++int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec); ++int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 1fd9e5a8..002431d8 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -87,8 +87,11 @@ struct readonly_default_oci_spec { + + static struct readonly_default_oci_spec g_rdspec; + +-static int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) + { ++ if (oci_spec == NULL) { ++ return -1; ++ } + if (oci_spec->annotations == NULL) { + oci_spec->annotations = util_common_calloc_s(sizeof(json_map_string_string)); + if (oci_spec->annotations == NULL) { +@@ -464,10 +467,14 @@ out: + return ret; + } + +-static int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_linux_resources_cpu(oci_runtime_spec *oci_spec) + { + int ret = 0; + ++ if (oci_spec == NULL) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -589,10 +596,14 @@ out: + return ret; + } + +-static int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_linux_resources_mem(oci_runtime_spec *oci_spec) + { + int ret = 0; + ++ if (oci_spec == NULL) { ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -731,8 +742,11 @@ out: + return ret; + } + +-static int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec) ++int make_sure_oci_spec_hooks(oci_runtime_spec *oci_spec) + { ++ if (oci_spec == NULL) { ++ return -1; ++ } + if (oci_spec->hooks == NULL) { + oci_spec->hooks = util_common_calloc_s(sizeof(oci_runtime_spec_hooks)); + if (oci_spec->hooks == NULL) { +@@ -2827,6 +2841,11 @@ int spec_add_linux_resources_hugepage_limit(oci_runtime_spec *oci_spec, const ch + int ret = 0; + defs_resources_hugepage_limits_element *hugepage_limit = NULL; + ++ if (oci_spec == NULL || page_size == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +@@ -2859,6 +2878,11 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type + int ret = 0; + defs_process_rlimits_element *rlimit = NULL; + ++ if (oci_spec == NULL || type == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ + ret = make_sure_oci_spec_linux_resources(oci_spec); + if (ret < 0) { + return -1; +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 7f0640df..874662cf 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -46,7 +46,6 @@ public: + + auto GetSockpath(std::vector &paths) -> bool; + +- // Stop plugins. + auto StopPlugins() -> bool; + + void RemoveClosedPlugins(); +@@ -58,19 +57,23 @@ public: + auto RunPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto StopPodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; + auto RemovePodSandbox(std::shared_ptr sandbox, Errors &error) ->bool; +- auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool; +- auto PostCreateContainer(const std::string &conId, Errors &error) ->bool; +- auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, Errors &error) -> bool; +- auto StartContainer(const std::string &conId, Errors &error) ->bool; +- auto PostStartContainer(const std::string &conId, Errors &error) ->bool; +- auto UpdateContainer(const std::string &conId, Errors &error) ->bool; +- auto PostUpdateContainer(const std::string &conId, Errors &error) ->bool; +- auto StopContainer(const std::string &conId, Errors &error) ->bool; +- auto RemoveContainer(const std::string &conId, Errors &error) ->bool; +- auto StateChange(nri_state_change_event *evt, Errors &error) ->bool; +- auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) ->bool; ++ auto CreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, ++ Errors &error) -> bool; ++ auto PostCreateContainer(const std::string &conId, Errors &error) -> bool; ++ auto UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ Errors &error) -> bool; ++ auto StartContainer(const std::string &conId, Errors &error) -> bool; ++ auto PostStartContainer(const std::string &conId, Errors &error) -> bool; ++ auto UpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto StopContainer(const std::string &conId, Errors &error) -> bool; ++ auto RemoveContainer(const std::string &conId, Errors &error) -> bool; ++ auto StateChange(nri_state_change_event *evt, Errors &error) -> bool; ++ auto updateContainers(const nri_update_containers_request *req, nri_update_containers_response **resp) -> bool; + + auto NewExternalPlugin(int fd) -> bool; ++ + private: + NRIAdaptation() = default; + NRIAdaptation(const NRIAdaptation &other) = delete; +@@ -86,18 +89,25 @@ private: + auto SortPlugins() -> bool; + void GetClosedPlugins(std::vector &closedPlugin); + +- auto IsSupport() -> bool; ++ auto ApplyUpdates(const std::vector &update, std::vector &failed, ++ bool getFailed, Errors &error) -> bool; + +- auto ApplyUpdates(const std::vector &update, std::vector &failed, bool getFailed, +- Errors &error) -> bool; ++ auto IsSupport() -> bool; + +- auto NRIPodSandbox(const std::shared_ptr &sandbox, Errors& error) -> std::unique_ptr>; +- auto NRIContainerByConConfig(const std::shared_ptr &sandbox, const runtime::v1::ContainerConfig &containerConfig, Errors& error) -> std::unique_ptr>; +- auto NRIContainerByID(const std::string &id, Errors& error) -> std::unique_ptr>; ++ auto NRIPodSandbox(const std::shared_ptr &sandbox, ++ Errors &error) -> std::unique_ptr>; ++ auto NRIContainerByConConfig(const std::shared_ptr &sandbox, ++ const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr>; ++ auto NRIContainerByID(const std::string &id, Errors &error) -> std::unique_ptr>; + + auto GetNRIPluginConfigPath(void) -> std::string; + auto GetNRIPluginPath(void) -> std::string; + auto GetNRISockPath(void) -> std::string; ++ ++ void PluginsStateChange(nri_state_change_event *evt); ++ bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); ++ bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); ++ + private: + RWMutex m_mutex; + static std::atomic m_instance; +diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc +new file mode 100644 +index 00000000..ff9d67c1 +--- /dev/null ++++ b/src/daemon/nri/nri_helpers.cc +@@ -0,0 +1,93 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-07-13 ++ * Description: provide nri helpers functions ++ *********************************************************************************/ ++ ++#include "nri_helpers.h" ++ ++#include ++ ++#include "utils.h" ++#include "isulad_config.h" ++ ++namespace NRIHelpers { ++std::string MarkForRemoval(const std::string &key) ++{ ++ return "-" + key; ++} ++ ++auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool ++{ ++ __isula_auto_free char *plugin_path = NULL; ++ ++ plugin_path = conf_get_nri_plugin_config_path(); ++ if (plugin_path == NULL) { ++ return false; ++ } ++ std::string compleName = idx + "-" + name; ++ std::vector dropIns = { ++ std::string(plugin_path) + "/" + compleName + ".conf", ++ std::string(plugin_path) + "/" + name + ".conf" ++ }; ++ ++ for (const std::string &path : dropIns) { ++ char buf[MAX_BUFFER_SIZE + 1] = { 0 }; ++ __isula_auto_close int fd = util_open(path.c_str(), O_RDONLY, 0); ++ if (fd < 0) { ++ ERROR("Failed to open '%s'", path.c_str()); ++ return false; ++ } ++ int len = util_read_nointr(fd, buf, sizeof(buf) - 1); ++ if (len < 0) { ++ SYSERROR("Failed to read nri plugin config : %s", path.c_str()); ++ return false; ++ } ++ config = std::string(buf); ++ return true; ++ } ++ return true; ++} ++ ++void GenerateRandomExternalName(std::string &ret) ++{ ++ __isula_auto_free char *external_name = NULL; ++ ++ external_name = (char *)util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1)); ++ if (external_name == NULL) { ++ ERROR("Out of memory"); ++ return; ++ } ++ ++ if (util_generate_random_str(external_name, (size_t)CONTAINER_ID_MAX_LEN)) { ++ ERROR("Generate exec suffix failed"); ++ return; ++ } ++ ++ ret = std::string(external_name); ++} ++ ++bool CheckPluginIndex(const std::string &idx) ++{ ++ if (idx.length() != 2) { ++ ERROR("Invalid plugin index \"%s\", must be 2 digits", idx.c_str()); ++ return false; ++ } ++ ++ if (!std::isdigit(idx[0]) || !std::isdigit(idx[1])) { ++ ERROR("Invalid plugin index \"%s\", (not [0-9][0-9])", idx.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++}// namespace NRIHelpers +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +index 06ee8419..1a2f488e 100644 +--- a/src/daemon/nri/nri_helpers.h ++++ b/src/daemon/nri/nri_helpers.h +@@ -37,7 +37,7 @@ std::string MarkForRemoval(const std::string &key); + + auto GetPluginConfig(std::string &idx, std::string &name, std::string &config) -> bool; + +-std::string GenerateRandomExternalName(void); ++void GenerateRandomExternalName(std::string &ret); + + bool CheckPluginIndex(const std::string &idx); + +diff --git a/src/utils/cpputils/transform.cc b/src/utils/cpputils/transform.cc +index 51c154fb..ba8c1f7a 100644 +--- a/src/utils/cpputils/transform.cc ++++ b/src/utils/cpputils/transform.cc +@@ -97,7 +97,7 @@ auto RepeatedPtrFieldToCharArray(const google::protobuf::RepeatedPtrField +Date: Thu, 15 Aug 2024 19:21:19 +1400 +Subject: [PATCH 120/121] get realpath before ns mountpoint verification + +Signed-off-by: zhongtao +--- + .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 9 +++++++-- + .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 9 +++++++-- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 77faf48a..3ece885f 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -424,6 +424,7 @@ cleanup_sandbox: + + void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr sandbox, Errors &error) + { ++ char real_path[PATH_MAX] = { 0 }; + std::string networkMode = sandbox->GetNetMode(); + if (!namespace_is_cni(networkMode.c_str()) || !sandbox->GetNetworkReady()) { + return; +@@ -435,10 +436,14 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr int + { + Errors networkErr; ++ char real_path[PATH_MAX] = { 0 }; + + bool ready = GetNetworkReady(realSandboxID, networkErr); + if (hostNetwork || (!ready && networkErr.Empty())) { +@@ -848,10 +849,14 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + goto cleanup; + } + ++ if (realpath(netnsPath.c_str(), real_path) == NULL) { ++ ERROR("Failed to get %s realpath", netnsPath.c_str()); ++ } ++ + // If the network namespace is not mounted, the network has been cleaned up + // and there is no need to call the cni plugin. +- if (!util_detect_mounted(netnsPath.c_str())) { +- WARN("Network namespace %s not exist", netnsPath.c_str()); ++ if (strlen(real_path) != 0 && !util_detect_mounted(real_path)) { ++ ERROR("Network namespace %s not exist", real_path); + goto cleanup; + } + +-- +2.25.1 + diff --git a/0121-nri-impl-for-nri-plugin-and-adaption.patch b/0121-nri-impl-for-nri-plugin-and-adaption.patch new file mode 100644 index 0000000..fc13950 --- /dev/null +++ b/0121-nri-impl-for-nri-plugin-and-adaption.patch @@ -0,0 +1,4724 @@ +From 343be112a3d5cfd3857f166fb5dc2473534e64d7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 15 Aug 2024 15:19:25 +0800 +Subject: [PATCH 121/121] [nri] impl for nri plugin and adaption + +Signed-off-by: zhongtao +--- + src/cmd/isulad/main.c | 15 + + src/daemon/common/nri/nri_convert.cc | 148 ++- + src/daemon/common/nri/nri_convert.h | 7 +- + src/daemon/common/nri/nri_spec.c | 223 +++- + src/daemon/common/nri/nri_spec.h | 6 + + src/daemon/common/nri/nri_utils.c | 476 +++---- + src/daemon/common/nri/nri_utils.h | 14 +- + src/daemon/config/isulad_config.c | 2 +- + .../v1/v1_cri_container_manager_service.cc | 104 +- + .../v1/v1_cri_pod_sandbox_manager_service.cc | 37 +- + .../executor/container_cb/execution_create.c | 21 + + .../modules/service/service_container.c | 12 + + src/daemon/modules/spec/specs.c | 7 +- + src/daemon/nri/nri_adaption.cc | 1165 +++++++++++++++++ + src/daemon/nri/nri_adaption.h | 13 +- + src/daemon/nri/nri_helpers.cc | 21 + + src/daemon/nri/nri_helpers.h | 4 + + src/daemon/nri/nri_plugin_ops.cc | 123 ++ + src/daemon/nri/nri_plugin_ops.h | 6 +- + src/daemon/nri/nri_result.cc | 977 ++++++++++++++ + src/daemon/nri/nri_result.h | 17 +- + src/daemon/nri/plugin.cc | 417 ++++++ + src/daemon/nri/plugin.h | 20 +- + src/utils/cutils/utils.c | 2 +- + 24 files changed, 3527 insertions(+), 310 deletions(-) + create mode 100644 src/daemon/nri/nri_adaption.cc + create mode 100644 src/daemon/nri/nri_plugin_ops.cc + create mode 100644 src/daemon/nri/nri_result.cc + create mode 100644 src/daemon/nri/plugin.cc + +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index 52ac3172..0228caa8 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -86,6 +86,9 @@ + #ifdef ENABLE_CDI + #include "cdi_operate_api.h" + #endif /* ENABLE_CDI */ ++#ifdef ENABLE_NRI ++#include "nri_plugin_ops.h" ++#endif + + sem_t g_daemon_shutdown_sem; + sem_t g_daemon_wait_shutdown_sem; +@@ -289,6 +292,11 @@ static void daemon_shutdown() + EVENT("Network module exit completed"); + #endif + ++#ifdef ENABLE_NRI ++ nri_adaption_shutdown(); ++ EVENT("nri module exit completed"); ++#endif ++ + clean_residual_files(); + EVENT("Clean residual files completed"); + +@@ -1834,6 +1842,13 @@ int main(int argc, char **argv) + } + #endif + ++#ifdef ENABLE_NRI ++ if (!nri_adaption_init()) { ++ ERROR("Failed to init nri adaption"); ++ goto failure; ++ } ++#endif ++ + clock_gettime(CLOCK_MONOTONIC, &t_end); + use_time = (double)(t_end.tv_sec - t_start.tv_sec) * (double)1000000000 + (double)(t_end.tv_nsec - t_start.tv_nsec); + use_time /= 1000000000; +diff --git a/src/daemon/common/nri/nri_convert.cc b/src/daemon/common/nri/nri_convert.cc +index 7cce64ec..30caf1dd 100644 +--- a/src/daemon/common/nri/nri_convert.cc ++++ b/src/daemon/common/nri/nri_convert.cc +@@ -20,6 +20,7 @@ + #include "path.h" + #include "transform.h" + #include "nri_utils.h" ++#include "cstruct_wrapper.h" + + static int64_t DefaultOOMScoreAdj = 0; + +@@ -142,22 +143,28 @@ static auto NRILinuxResourcesFromCRI(const runtime::v1::LinuxContainerResources + + static auto NRILinuxFromCRI(const runtime::v1::LinuxPodSandboxConfig &config, nri_linux_pod_sandbox &linux) -> bool + { +- if (!init_nri_linux_resources(&linux.pod_overhead)) { +- ERROR("Failed to init nri linux overhead resources for pod"); +- return false; +- } +- if (!init_nri_linux_resources(&linux.pod_resources)) { +- ERROR("Failed to init nri linux resources resources for pod"); +- return false; +- } +- if (config.has_overhead() && !NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { +- ERROR("Failed to transform overhead to nri for pod"); +- return false; ++ if (config.has_overhead()) { ++ linux.pod_overhead = init_nri_linux_resources(); ++ if (linux.pod_overhead == nullptr) { ++ ERROR("Failed to init nri linux overhead resources for pod"); ++ return false; ++ } ++ if (!NRILinuxResourcesFromCRI(config.overhead(), *linux.pod_overhead)) { ++ ERROR("Failed to transform overhead to nri for pod"); ++ return false; ++ } + } + +- if (config.has_resources() && !NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { +- ERROR("Failed to transform resources to nri for pod"); +- return false; ++ if (config.has_resources()) { ++ linux.pod_resources = init_nri_linux_resources(); ++ if (linux.pod_resources == nullptr) { ++ ERROR("Failed to init nri linux resources resources for pod"); ++ return false; ++ } ++ if (!NRILinuxResourcesFromCRI(config.resources(), *linux.pod_resources)) { ++ ERROR("Failed to transform resources to nri for pod"); ++ return false; ++ } + } + + linux.cgroup_parent = util_strdup_s(config.cgroup_parent().c_str()); +@@ -507,13 +514,13 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon + } + + if (src->cpu != nullptr) { +- if (src->cpu->shares != NULL) { ++ if (src->cpu->shares != nullptr) { + resources.set_cpu_shares(*src->cpu->shares); + } +- if (src->cpu->quota != NULL) { ++ if (src->cpu->quota != nullptr) { + resources.set_cpu_quota(*src->cpu->quota); + } +- if (src->cpu->period != NULL) { ++ if (src->cpu->period != nullptr) { + resources.set_cpu_period(*src->cpu->period); + } + +@@ -535,5 +542,112 @@ auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxCon + Transform::JsonMapToProtobufMapForString(src->unified, *resources.mutable_unified()); + } + ++ return true; ++} ++ ++auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources * ++{ ++ nri_linux_resources *resources = nullptr; ++ ++ resources = init_nri_linux_resources(); ++ if (resources == nullptr) { ++ ERROR("Failed to init nri linux resources"); ++ return nullptr; ++ } ++ ++ resources->cpu->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (resources->cpu->shares == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->shares) = src.cpu_shares(); ++ ++ resources->cpu->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (resources->cpu->quota == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->quota) = src.cpu_quota(); ++ ++ resources->cpu->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (resources->cpu->period == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->cpu->period) = src.cpu_period(); ++ ++ resources->cpu->cpus = util_strdup_s(src.cpuset_cpus().c_str()); ++ resources->cpu->mems = util_strdup_s(src.cpuset_mems().c_str()); ++ ++ resources->memory->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (resources->memory->limit == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ *(resources->memory->limit) = src.memory_limit_in_bytes(); ++ ++ resources->hugepage_limits = (nri_hugepage_limit **)util_smart_calloc_s(sizeof(nri_hugepage_limit *), ++ src.hugepage_limits_size()); ++ if (resources->hugepage_limits == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ ++ for (int i = 0; i < src.hugepage_limits_size(); i++) { ++ resources->hugepage_limits[i] = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (resources->hugepage_limits[i] == nullptr) { ++ ERROR("Out of memory"); ++ goto error_out; ++ } ++ resources->hugepage_limits[i]->page_size = util_strdup_s(src.hugepage_limits(i).page_size().c_str()); ++ resources->hugepage_limits[i]->limit = src.hugepage_limits(i).limit(); ++ resources->hugepage_limits_len++; ++ } ++ ++ return resources; ++ ++error_out: ++ free_nri_linux_resources(resources); ++ resources = nullptr; ++ return resources; ++} ++ ++auto PodSandboxesToNRI(const std::vector> &arrs, ++ std::vector &pods) -> bool ++{ ++ size_t i = 0; ++ for (i = 0; i < arrs.size(); i++) { ++ nri_pod_sandbox *pod = (nri_pod_sandbox *)util_common_calloc_s(sizeof(nri_pod_sandbox)); ++ if (pod == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!PodSandboxToNRI(arrs[i], *pod)) { ++ ERROR("Failed to transform pod to nri for pod : %s", arrs[i]->GetName().c_str()); ++ return false; ++ } ++ pods.push_back(pod); ++ } ++ ++ return true; ++} ++ ++auto ContainersToNRI(std::vector> &containers, ++ std::vector &cons) -> bool ++{ ++ size_t i = 0; ++ for (i = 0; i < containers.size(); i++) { ++ nri_container *con = (nri_container *)util_common_calloc_s(sizeof(nri_container)); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ if (!ContainerToNRIByID(containers[i].get()->id(), *con)) { ++ ERROR("Failed to transform container to nri for container : %s", containers[i]->metadata().name().c_str()); ++ return false; ++ } ++ cons.push_back(con); ++ } ++ + return true; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_convert.h b/src/daemon/common/nri/nri_convert.h +index c04b14e4..a0caf6ce 100644 +--- a/src/daemon/common/nri/nri_convert.h ++++ b/src/daemon/common/nri/nri_convert.h +@@ -30,8 +30,11 @@ + auto PodSandboxToNRI(const std::shared_ptr &sandbox, nri_pod_sandbox &pod) -> bool; + auto ContainerToNRIByConConfig(const runtime::v1::ContainerConfig &containerConfig, nri_container &con) -> bool; + auto ContainerToNRIByID(const std::string &id, nri_container &con) -> bool; +-auto PodSandboxesToNRI(const std::vector> &arrs, nri_pod_sandbox **pod, +- int pod_len) -> bool; ++auto PodSandboxesToNRI(const std::vector> &arrs, ++ std::vector &pods) -> bool; ++auto ContainersToNRI(std::vector> &containers, ++ std::vector &cons) -> bool; + + auto LinuxResourcesFromNRI(const nri_linux_resources *src, runtime::v1::LinuxContainerResources &resources) -> bool; ++auto LinuxResourcesToNRI(const runtime::v1::LinuxContainerResources &src) -> nri_linux_resources *; + #endif // DAEMON_COMMON_NRI_NRI_CONVERT_H +diff --git a/src/daemon/common/nri/nri_spec.c b/src/daemon/common/nri/nri_spec.c +index 855fe3b3..327d31e1 100644 +--- a/src/daemon/common/nri/nri_spec.c ++++ b/src/daemon/common/nri/nri_spec.c +@@ -176,6 +176,7 @@ static int nri_adjust_annotation(const nri_container_adjustment *adjust, oci_run + + free_json_map_string_string(oci_spec->annotations); + oci_spec->annotations = cleard; ++ cleard = NULL; + ret = 0; + + free_out: +@@ -330,7 +331,34 @@ static int nri_adjust_hooks(const nri_container_adjustment *adjust, oci_runtime_ + return ret; + } + +-static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++static int host_spec_add_device(host_config *spec, defs_device *device) ++{ ++ size_t i; ++ ++ if (device == NULL) { ++ return -1; ++ } ++ ++ for (i = 0; i < spec->nri_devices_len; i++) { ++ if (strcmp(spec->nri_devices[i]->path, device->path) == 0) { ++ free_defs_device(spec->nri_devices[i]); ++ spec->nri_devices[i] = device; ++ return 0; ++ } ++ } ++ ++ if (util_mem_realloc((void **)&spec->nri_devices, (spec->nri_devices_len + 1) * sizeof(defs_device *), ++ (void *)spec->nri_devices, spec->nri_devices_len * sizeof(defs_device *)) != 0) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ spec->nri_devices[spec->nri_devices_len] = device; ++ spec->nri_devices_len++; ++ ++ return 0; ++} ++ ++static int nri_adjust_oci_spec_devices(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { + return 0; +@@ -349,6 +377,25 @@ static int nri_adjust_devices(const nri_container_adjustment *adjust, oci_runtim + return 0; + } + ++static int nri_adjust_host_spec_devices(const nri_container_adjustment *adjust, host_config *spec) ++{ ++ if (adjust->linux == NULL || adjust->linux->devices == NULL || adjust->linux->devices_len == 0) { ++ return 0; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < adjust->linux->devices_len; i++) { ++ nri_linux_device *dev = adjust->linux->devices[i]; ++ if (host_spec_add_device(spec, nri_device_to_oci(dev)) != 0) { ++ ERROR("Failed to add device %s", dev->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + static int nri_adjust_cgroup_path(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->linux == NULL || adjust->linux->cgroups_path == NULL) { +@@ -500,7 +547,50 @@ static int nri_adjust_mounts(const nri_container_adjustment *adjust, oci_runtime + return 0; + } + +-static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) ++static int host_spec_add_linux_resources_rlimit(host_config *spec, const char *type, uint64_t hard, uint64_t soft) ++{ ++ size_t j; ++ bool exists = false; ++ host_config_nri_rlimits_element *rlimit = NULL; ++ ++ if (spec == NULL || type == NULL) { ++ ERROR("Invalid arguments"); ++ return -1; ++ } ++ ++ rlimit = util_common_calloc_s(sizeof(host_config_nri_rlimits_element)); ++ if (rlimit == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ rlimit->type = util_strdup_s(type); ++ rlimit->hard = hard; ++ rlimit->soft = soft; ++ ++ for (j = 0; j < spec->nri_rlimits_len; j++) { ++ if (spec->nri_rlimits[j]->type == NULL) { ++ ERROR("rlimit type is empty"); ++ free_host_config_nri_rlimits_element(rlimit); ++ return -1; ++ } ++ if (strcmp(spec->nri_rlimits[j]->type, rlimit->type) == 0) { ++ exists = true; ++ break; ++ } ++ } ++ if (exists) { ++ /* override ulimit */ ++ free_host_config_nri_rlimits_element(spec->nri_rlimits[j]); ++ spec->nri_rlimits[j] = rlimit; ++ } else { ++ spec->nri_rlimits[spec->nri_rlimits_len] = rlimit; ++ spec->nri_rlimits_len++; ++ } ++ ++ return 0; ++} ++ ++static int nri_adjust_oci_spec_rlimit(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { + if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { + return 0; +@@ -522,6 +612,36 @@ static int nri_adjust_rlimit(const nri_container_adjustment *adjust, oci_runtime + return 0; + } + ++static int nri_adjust_host_spec_rlimit(const nri_container_adjustment *adjust, host_config *spec) ++{ ++ if (adjust->rlimits == NULL || adjust->rlimits_len == 0) { ++ return 0; ++ } ++ ++ if (spec->nri_rlimits == NULL) { ++ spec->nri_rlimits = util_common_calloc_s(sizeof(host_config_nri_rlimits_element *) * adjust->rlimits_len); ++ if (spec->nri_rlimits == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ } ++ ++ size_t i; ++ for (i = 0; i < adjust->rlimits_len; i++) { ++ nri_posix_rlimit *rlimit = adjust->rlimits[i]; ++ if (rlimit->type == NULL) { ++ ERROR("Invalid rlimit type"); ++ return -1; ++ } ++ if (host_spec_add_linux_resources_rlimit(spec, rlimit->type, rlimit->soft, rlimit->hard) != 0) { ++ ERROR("Failed to add rlimit"); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + // todo: we do not support it blockio_class + static int nri_adjust_blockio_class(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec) + { +@@ -554,7 +674,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + +- if (nri_adjust_devices(adjust, oci_spec) != 0) { ++ if (nri_adjust_oci_spec_devices(adjust, oci_spec) != 0) { + ERROR("Failed to do nri adjust devices in oci spec"); + return -1; + } +@@ -580,7 +700,7 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + +- if (nri_adjust_rlimit(adjust, oci_spec) != 0) { ++ if (nri_adjust_oci_spec_rlimit(adjust, oci_spec) != 0) { + ERROR("Failed to do nri adjust rlimit in oci spec"); + return -1; + } +@@ -598,5 +718,100 @@ int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec + return -1; + } + ++ return 0; ++} ++ ++int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec) ++{ ++ if (adjust == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (nri_adjust_host_spec_devices(adjust, host_spec) != 0) { ++ ERROR("Failed to do nri adjust devices in host config"); ++ return -1; ++ } ++ ++ if (nri_adjust_host_spec_rlimit(adjust, host_spec) != 0) { ++ ERROR("Failed to do nri adjust rlimit in host config"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static defs_device *copy_def_devices(const defs_device *dev) ++{ ++ defs_device *tmp_dev = util_common_calloc_s(sizeof(defs_device)); ++ if (tmp_dev == NULL) { ++ return NULL; ++ } ++ tmp_dev->type = util_strdup_s(dev->type); ++ tmp_dev->path = util_strdup_s(dev->path); ++ tmp_dev->file_mode = dev->file_mode; ++ tmp_dev->major = dev->major; ++ tmp_dev->minor = dev->minor; ++ tmp_dev->uid = dev->uid; ++ tmp_dev->gid = dev->gid; ++ return tmp_dev; ++} ++ ++static int merge_nri_devices(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ for (i = 0; i < host_spec->nri_devices_len; i++) { ++ if (spec_add_device(oci_spec, copy_def_devices(host_spec->nri_devices[i])) != 0) { ++ ERROR("Failed to add device %s", host_spec->nri_devices[i]->path); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int merge_nri_ulimits(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ size_t i; ++ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ for (i = 0; i < host_spec->nri_rlimits_len; i++) { ++ host_config_nri_rlimits_element *rlimit = host_spec->nri_rlimits[i]; ++ if (spec_add_linux_resources_rlimit(oci_spec, rlimit->type, rlimit->hard, rlimit->soft) != 0) { ++ ERROR("Failed to add rlimit %s", rlimit->type); ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec) ++{ ++ if (oci_spec == NULL || host_spec == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (merge_nri_devices(oci_spec, host_spec) != 0) { ++ ERROR("Failed to merge nri devices"); ++ return -1; ++ } ++ ++ if (merge_nri_ulimits(oci_spec, host_spec) != 0) { ++ ERROR("Failed to merge nri ulimits"); ++ return -1; ++ } ++ + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_spec.h b/src/daemon/common/nri/nri_spec.h +index e7c5035d..1a622284 100644 +--- a/src/daemon/common/nri/nri_spec.h ++++ b/src/daemon/common/nri/nri_spec.h +@@ -18,7 +18,13 @@ + + #include + #include ++#include + + int nri_adjust_oci_spec(const nri_container_adjustment *adjust, oci_runtime_spec *oci_spec); + ++// the device and ulimit will be updated when starting, so they need to be stored in host-spec ++int nri_adjust_host_spec(const nri_container_adjustment *adjust, host_config *host_spec); ++ ++int update_oci_nri(oci_runtime_spec *oci_spec, host_config *host_spec); ++ + #endif // DAEMON_COMMON_NRI_NRI_SPEC_H +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.c b/src/daemon/common/nri/nri_utils.c +index 51054e32..d2b62117 100644 +--- a/src/daemon/common/nri/nri_utils.c ++++ b/src/daemon/common/nri/nri_utils.c +@@ -19,218 +19,235 @@ + + #include "utils.h" + +-static bool copy_nri_hugepage_limit(const nri_hugepage_limit* src, nri_hugepage_limit** dest) ++static nri_hugepage_limit*copy_nri_hugepage_limit(const nri_hugepage_limit *src) + { +- if (src == NULL || dest == NULL) { ++ nri_hugepage_limit *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); +- if (*dest == NULL) { ++ dest = (nri_hugepage_limit *)util_common_calloc_s(sizeof(nri_hugepage_limit)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*dest)->limit = src->limit; +- (*dest)->page_size = util_strdup_s(src->page_size); +- return true; ++ dest->limit = src->limit; ++ dest->page_size = util_strdup_s(src->page_size); ++ return dest; + } + +-static bool copy_nri_hook(const nri_hook *src, nri_hook **dest) ++static nri_hook *copy_nri_hook(const nri_hook *src) + { +- if (src == NULL || dest == NULL) { ++ nri_hook *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); ++ dest = (nri_hook *)util_common_calloc_s(sizeof(nri_hook)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->args = util_copy_array_by_len(src->args, src->args_len); +- (*dest)->args_len = src->args_len; +- (*dest)->env = util_copy_array_by_len(src->env, src->env_len); +- (*dest)->env_len = src->env_len; +- (*dest)->path = util_strdup_s(src->path); +- return true; ++ dest->args = util_copy_array_by_len(src->args, src->args_len); ++ dest->args_len = src->args_len; ++ dest->env = util_copy_array_by_len(src->env, src->env_len); ++ dest->env_len = src->env_len; ++ dest->path = util_strdup_s(src->path); ++ return dest; + } + +-static bool copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src, nri_linux_device_cgroup **dest) ++static nri_linux_device_cgroup *copy_nri_linux_device_cgroup(const nri_linux_device_cgroup *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_device_cgroup *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); ++ dest = (nri_linux_device_cgroup *)util_common_calloc_s(sizeof(nri_linux_device_cgroup)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->allow = src->allow; +- (*dest)->type = util_strdup_s(src->type); +- (*dest)->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->major == NULL) { ++ dest->allow = src->allow; ++ dest->type = util_strdup_s(src->type); ++ dest->major = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->major == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- (*dest)->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->minor == NULL) { ++ dest->minor = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->minor == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- (*dest)->access = util_strdup_s(src->access); +- return true; ++ dest->access = util_strdup_s(src->access); ++ return dest; ++free_out: ++ free_nri_linux_device_cgroup(dest); ++ return NULL; + } + +-static bool copy_nri_linux_cpu(const nri_linux_cpu *src, nri_linux_cpu **dest) ++static nri_linux_cpu *copy_nri_linux_cpu(const nri_linux_cpu *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_cpu *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- (*dest) = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); +- if ((*dest) == NULL) { ++ dest = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->cpus = util_strdup_s(src->cpus); +- (*dest)->mems = util_strdup_s(src->mems); ++ dest->cpus = util_strdup_s(src->cpus); ++ dest->mems = util_strdup_s(src->mems); + if (src->period != NULL) { +- (*dest)->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->period == NULL) { ++ dest->period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->period == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->period = *src->period; ++ *dest->period = *src->period; + } + + if (src->quota != NULL) { +- (*dest)->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->quota == NULL) { ++ dest->quota = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->quota == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->quota = *src->quota; ++ *dest->quota = *src->quota; + } + + if (src->realtime_period != NULL) { +- (*dest)->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->realtime_period == NULL) { ++ dest->realtime_period = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->realtime_period == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->realtime_period = *src->realtime_period; ++ *dest->realtime_period = *src->realtime_period; + } + + if (src->realtime_runtime != NULL) { +- (*dest)->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->realtime_runtime == NULL) { ++ dest->realtime_runtime = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->realtime_runtime == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->realtime_runtime = *src->realtime_runtime; ++ *dest->realtime_runtime = *src->realtime_runtime; + } + + if (src->shares != NULL) { +- (*dest)->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->shares == NULL) { ++ dest->shares = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->shares == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->shares = *src->shares; ++ *dest->shares = *src->shares; + } + +- return true; ++ return dest; ++ ++free_out: ++ free_nri_linux_cpu(dest); ++ return NULL; + } + +-static bool copy_nri_linux_memory(const nri_linux_memory *src, nri_linux_memory **dest) ++static nri_linux_memory *copy_nri_linux_memory(const nri_linux_memory *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_memory *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ ++ dest = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } + if (src->limit != NULL) { +- (*dest)->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->limit == NULL) { ++ dest->limit = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->limit == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->limit = *src->limit; ++ *dest->limit = *src->limit; + } + + if (src->reservation != NULL) { +- (*dest)->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->reservation == NULL) { ++ dest->reservation = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->reservation == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->reservation = *src->reservation; ++ *dest->reservation = *src->reservation; + } + + if (src->swap != NULL) { +- (*dest)->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->swap == NULL) { ++ dest->swap = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->swap == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->swap = *src->swap; ++ *dest->swap = *src->swap; + } + + if (src->kernel != NULL) { +- (*dest)->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->kernel == NULL) { ++ dest->kernel = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->kernel == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->kernel = *src->kernel; ++ *dest->kernel = *src->kernel; + } + + + if (src->kernel_tcp != NULL) { +- (*dest)->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); +- if ((*dest)->kernel_tcp == NULL) { ++ dest->kernel_tcp = (int64_t *)util_common_calloc_s(sizeof(int64_t)); ++ if (dest->kernel_tcp == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->kernel_tcp = *src->kernel_tcp; ++ *dest->kernel_tcp = *src->kernel_tcp; + } + + if (src->swappiness != NULL) { +- (*dest)->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); +- if ((*dest)->swappiness == NULL) { ++ dest->swappiness = (uint64_t *)util_common_calloc_s(sizeof(uint64_t)); ++ if (dest->swappiness == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->swappiness = *src->swappiness; ++ *dest->swappiness = *src->swappiness; + } + + if (src->disable_oom_killer != NULL) { +- (*dest)->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); +- if ((*dest)->disable_oom_killer == NULL) { ++ dest->disable_oom_killer = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (dest->disable_oom_killer == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->disable_oom_killer = *src->disable_oom_killer; ++ *dest->disable_oom_killer = *src->disable_oom_killer; + } + + if (src->use_hierarchy != NULL) { +- (*dest)->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); +- if ((*dest)->use_hierarchy == NULL) { ++ dest->use_hierarchy = (uint8_t *)util_common_calloc_s(sizeof(uint8_t)); ++ if (dest->use_hierarchy == NULL) { + ERROR("Out of memory"); +- return false; ++ goto free_out; + } +- *(*dest)->use_hierarchy = *src->use_hierarchy; ++ *dest->use_hierarchy = *src->use_hierarchy; + } +- return true; ++ return dest; ++ ++free_out: ++ free_nri_linux_memory(dest); ++ return NULL; + } + + bool is_marked_for_removal(const char* key, char **out) +@@ -241,7 +258,7 @@ bool is_marked_for_removal(const char* key, char **out) + } + + if (!util_has_prefix(key, "-")) { +- *out = (char*)key; ++ *out = util_strdup_s(key); + return false; + } + +@@ -254,94 +271,153 @@ bool is_marked_for_removal(const char* key, char **out) + return true; + } + +-bool copy_nri_mount(const nri_mount *src, nri_mount **dest) ++nri_mount *copy_nri_mount(const nri_mount *src) + { +- if (src == NULL || dest == NULL) { ++ nri_mount *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); ++ ++ dest = (nri_mount *)util_common_calloc_s(sizeof(nri_mount)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->destination = util_strdup_s(src->destination); +- (*dest)->options = util_copy_array_by_len(src->options, src->options_len); +- (*dest)->options_len = src->options_len; +- (*dest)->source = util_strdup_s(src->source); +- (*dest)->type = util_strdup_s(src->type); +- return true; ++ dest->destination = util_strdup_s(src->destination); ++ dest->options = util_copy_array_by_len(src->options, src->options_len); ++ dest->options_len = src->options_len; ++ dest->source = util_strdup_s(src->source); ++ dest->type = util_strdup_s(src->type); ++ return dest; + } + +-bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest) ++nri_linux_device *copy_nri_device(const nri_linux_device *src) + { +- if (src == NULL || dest == NULL) { ++ nri_linux_device *dest = NULL; ++ ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); ++ ++ dest = (nri_linux_device *)util_common_calloc_s(sizeof(nri_linux_device)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->key = util_strdup_s(src->key); +- (*dest)->value = util_strdup_s(src->value); +- return true; ++ if (src->file_mode != NULL) { ++ dest->file_mode = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->file_mode == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->file_mode = *src->file_mode; ++ } ++ ++ if (src->uid != NULL) { ++ dest->uid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->uid == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->uid = *src->uid; ++ } ++ ++ if (src->gid != NULL) { ++ dest->gid = (uint32_t *)util_common_calloc_s(sizeof(uint32_t)); ++ if (dest->gid == NULL) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ *dest->gid = *src->gid; ++ } ++ ++ dest->major = src->major; ++ dest->minor = src->minor; ++ dest->path = util_strdup_s(src->path); ++ dest->type = util_strdup_s(src->type); ++ ++ return dest; ++free_out: ++ free_nri_linux_device(dest); ++ return NULL; + } + +-bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest) ++nri_key_value *copy_nri_key_value(const nri_key_value *src) + { +- if (src == NULL || dest == NULL) { ++ nri_key_value *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- *dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ dest = (nri_key_value *)util_common_calloc_s(sizeof(nri_key_value)); + if (dest == NULL) { + ERROR("Out of memory"); + return false; + } +- (*dest)->hard = src->hard; +- (*dest)->soft = src->soft; +- (*dest)->type = util_strdup_s(src->type); +- return true; ++ dest->key = util_strdup_s(src->key); ++ dest->value = util_strdup_s(src->value); ++ return dest; + } + +-bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest) ++nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src) + { +- if (src == NULL || dest == NULL) { ++ nri_posix_rlimit *dest = NULL; ++ if (src == NULL) { + ERROR("Invalid input arguments"); + return false; + } +- +- *dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if (*dest == NULL) { ++ dest = (nri_posix_rlimit *)util_common_calloc_s(sizeof(nri_posix_rlimit)); ++ if (dest == NULL) { + ERROR("Out of memory"); + return false; + } ++ dest->hard = src->hard; ++ dest->soft = src->soft; ++ dest->type = util_strdup_s(src->type); ++ return dest; ++} + +- if (!init_nri_linux_resources(dest)) { +- ERROR("Failed to init dest nri linux resources"); +- goto free_out; ++nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src) ++{ ++ nri_linux_resources *dest = NULL; ++ if (src == NULL) { ++ ERROR("Invalid input arguments"); ++ return false; + } + +- if (!copy_nri_linux_cpu(src->cpu, &(*dest)->cpu)) { +- ERROR("Failed to copy nri_linux_cpu"); +- goto free_out; ++ dest = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (dest == NULL) { ++ ERROR("Out of memory"); ++ return false; + } + +- if (!copy_nri_linux_memory(src->memory, &(*dest)->memory)) { +- ERROR("Failed to copy nri_linux_memory"); +- goto free_out; ++ if (src->cpu != NULL) { ++ dest->cpu = copy_nri_linux_cpu(src->cpu); ++ if (dest->cpu == NULL) { ++ ERROR("Failed to copy nri_linux_cpu"); ++ goto free_out; ++ } + } + +- (*dest)->blockio_class = util_strdup_s(src->blockio_class); +- (*dest)->rdt_class = util_strdup_s(src->rdt_class); ++ if (src->memory != NULL) { ++ dest->memory = copy_nri_linux_memory(src->memory); ++ if (dest->memory == NULL) { ++ ERROR("Failed to copy nri_linux_memory"); ++ goto free_out; ++ } ++ } ++ ++ dest->blockio_class = util_strdup_s(src->blockio_class); ++ dest->rdt_class = util_strdup_s(src->rdt_class); + + if (src->hugepage_limits_len > 0) { +- (*dest)->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), ++ dest->hugepage_limits = (nri_hugepage_limit**)util_smart_calloc_s(sizeof(nri_hugepage_limit*), + src->hugepage_limits_len); +- for (size_t i = 0; i < src->hugepage_limits_len; ++i) { +- if (!copy_nri_hugepage_limit(src->hugepage_limits[i], &((*dest)->hugepage_limits[i]))) { ++ for (size_t i = 0; i < src->hugepage_limits_len; i++) { ++ dest->hugepage_limits[i] = copy_nri_hugepage_limit(src->hugepage_limits[i]); ++ if (dest->hugepage_limits[i] == NULL) { + ERROR("Failed to copy nri_hugepage_limit"); + goto free_out; + } +@@ -349,25 +425,26 @@ bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resource + } + + if (src->devices_len > 0) { +- (*dest)->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); +- for (size_t i = 0; i < src->devices_len; ++i) { +- if (!copy_nri_linux_device_cgroup(src->devices[i], &((*dest)->devices[i]))) { ++ dest->devices = (nri_linux_device_cgroup**)util_smart_calloc_s(sizeof(nri_linux_device_cgroup*), src->devices_len); ++ for (size_t i = 0; i < src->devices_len; i++) { ++ dest->devices[i] = copy_nri_linux_device_cgroup(src->devices[i]); ++ if (dest->devices[i] == NULL) { + ERROR("Failed to copy nri_linux_device_cgroup"); + goto free_out; + } + } + } + +- if (dup_json_map_string_string(src->unified, (*dest)->unified)) { ++ if (dup_json_map_string_string(src->unified, dest->unified)) { + ERROR("Failed to copy json_map_string_string"); + goto free_out; + } + +- return true; ++ return dest; + + free_out: +- free_nri_linux_resources(*dest); +- return false; ++ free_nri_linux_resources(dest); ++ return NULL; + } + + bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, +@@ -381,140 +458,67 @@ bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook * + return false; + } + +- if (util_mem_realloc((void**)&targetHooks, newSize, (void**)&targetHooks, oldSize) != 0) { ++ if (util_mem_realloc((void**)&targetHooks, newSize, targetHooks, oldSize) != 0) { + ERROR("Failed to realloc and assign hook array"); + return false; + } + + for (size_t i = 0; i < sourceLen; i++) { +- if (!copy_nri_hook(sourceHooks[i], &targetHooks[targetSize++])) { ++ targetHooks[targetSize] = copy_nri_hook(sourceHooks[i]); ++ if (targetHooks[targetSize] == NULL) { + ERROR("Failed to copy hook"); + return false; + } ++ targetSize++; + } + + return true; + } + +-bool init_nri_container_adjust(nri_container_adjustment **adjust) ++nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure) + { +- if (adjust == NULL) { ++ nri_container_update *update = NULL; ++ if (id == NULL) { + ERROR("Invalid input arguments"); + return false; + } + +- *adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); +- if (*adjust == NULL) { ++ update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); ++ if (update == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*adjust)->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); +- if ((*adjust)->annotations == NULL) { +- goto free_out; +- } +- +- (*adjust)->env = (nri_key_value **)util_common_calloc_s(sizeof(nri_key_value *)); +- if ((*adjust)->env == NULL) { +- goto free_out; +- } +- (*adjust)->env_len = 0; +- +- (*adjust)->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); +- if ((*adjust)->hooks == NULL) { +- goto free_out; +- } +- +- (*adjust)->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); +- if ((*adjust)->linux == NULL) { +- goto free_out; +- } +- +- (*adjust)->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if ((*adjust)->linux->resources == NULL) { +- ERROR("Out of memory"); +- return false; +- } +- +- (*adjust)->mounts = (nri_mount **)util_common_calloc_s(sizeof(nri_mount *)); +- if ((*adjust)->mounts == NULL) { +- goto free_out; +- } +- (*adjust)->mounts_len = 0; +- +- (*adjust)->rlimits = (nri_posix_rlimit **)util_common_calloc_s(sizeof(nri_posix_rlimit *)); +- if ((*adjust)->rlimits == NULL) { +- goto free_out; +- } +- (*adjust)->rlimits_len = 0; ++ update->container_id = util_strdup_s(id); + +- return true; +- +-free_out: +- ERROR("Out of memory"); +- free_nri_container_adjustment(*adjust); +- return false; ++ update->ignore_failure = ignore_failure; ++ return update; + } + +-bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure) ++nri_linux_resources *init_nri_linux_resources() + { +- if (update == NULL || id == NULL) { +- ERROR("Invalid input arguments"); +- return false; +- } +- +- *update = (nri_container_update *)util_common_calloc_s(sizeof(nri_container_update)); +- if (*update == NULL) { +- ERROR("Out of memory"); +- return false; +- } +- +- (*update)->container_id = util_strdup_s(id); +- (*update)->linux = (nri_linux_container_update *)util_common_calloc_s(sizeof(nri_linux_container_update)); +- if ((*update)->linux == NULL) { +- goto free_out; +- } ++ nri_linux_resources *resources = NULL; + +- (*update)->ignore_failure = ignore_failure; +- return true; +- +-free_out: +- ERROR("Out of memory"); +- free_nri_container_update(*update); +- return false; +-} +- +-bool init_nri_linux_resources(nri_linux_resources **resources) +-{ ++ resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); + if (resources == NULL) { +- ERROR("Invalid input arguments"); +- return false; +- } +- +- *resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); +- if (*resources == NULL) { + ERROR("Out of memory"); + return false; + } + +- (*resources)->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); +- if ((*resources)->cpu == NULL) { ++ resources->cpu = (nri_linux_cpu *)util_common_calloc_s(sizeof(nri_linux_cpu)); ++ if (resources->cpu == NULL) { + goto free_out; + } + +- (*resources)->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); +- if ((*resources)->memory == NULL) { ++ resources->memory = (nri_linux_memory *)util_common_calloc_s(sizeof(nri_linux_memory)); ++ if (resources->memory == NULL) { + goto free_out; + } + +- (*resources)->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); +- if ((*resources)->unified == NULL) { +- goto free_out; +- } +- return true; ++ return resources; + + free_out: + ERROR("Out of memory"); +- free_nri_linux_resources(*resources); +- return false; ++ free_nri_linux_resources(resources); ++ return NULL; + } +\ No newline at end of file +diff --git a/src/daemon/common/nri/nri_utils.h b/src/daemon/common/nri/nri_utils.h +index 7bf54a71..171055df 100644 +--- a/src/daemon/common/nri/nri_utils.h ++++ b/src/daemon/common/nri/nri_utils.h +@@ -49,19 +49,19 @@ typedef enum { + LAST = 12, + } NRI_Event; + +-bool copy_nri_mount(const nri_mount *src, nri_mount **dest); +-bool copy_nri_key_value(const nri_key_value *src, nri_key_value **dest); +-bool copy_nri_posix_rlimit(const nri_posix_rlimit *src, nri_posix_rlimit **dest); +-bool copy_nri_linux_resources(const nri_linux_resources *src, nri_linux_resources **dest); ++nri_mount *copy_nri_mount(const nri_mount *src); ++nri_posix_rlimit *copy_nri_posix_rlimit(const nri_posix_rlimit *src); ++nri_linux_resources *copy_nri_linux_resources(const nri_linux_resources *src); ++nri_key_value *copy_nri_key_value(const nri_key_value *src); ++nri_linux_device *copy_nri_device(const nri_linux_device *src); + + bool is_marked_for_removal(const char* key, char **out); + + bool merge_nri_hooks(nri_hook **targetHooks, size_t targetSize, const nri_hook **sourceHooks, + size_t sourceLen); + +-bool init_nri_container_adjust(nri_container_adjustment **adjust); +-bool init_nri_container_update(nri_container_update **update, const char *id, uint8_t ignore_failure); +-bool init_nri_linux_resources(nri_linux_resources **resources); ++nri_container_update *init_nri_container_update(const char *id, const uint8_t ignore_failure); ++nri_linux_resources *init_nri_linux_resources(void); + + #ifdef __cplusplus + } +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 9ba1c8a0..020c9f32 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1938,7 +1938,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + override_string_value(&args->json_confs->plugin_path, &tmp_json_confs->plugin_path); + args->json_confs->plugin_registration_timeout = tmp_json_confs->plugin_registration_timeout; + args->json_confs->plugin_requst_timeout = tmp_json_confs->plugin_requst_timeout; +- override_string_value(&args->json_confs->nri_socket_path, &tmp_json_confs->nri_socket_path); ++ // Setting socket plugin path is not supported now + #endif + args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 1cee68ec..d3fdd76a 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -27,6 +27,11 @@ + #include "stream_server.h" + #include "sandbox_manager.h" + ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#include "nri_helpers.h" ++#endif ++ + namespace CRIV1 { + auto ContainerManagerService::GetContainerOrSandboxRuntime(const std::string &realID, Errors &error) -> std::string + { +@@ -505,11 +510,24 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + return response_id; + } + ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ nri_container_adjustment *adjust = NULL; ++ if (!NRIAdaptation::GetInstance()->CreateContainer(sandbox, response_id, containerConfig, &adjust, nriErr)) { ++ ERROR("Failed to get NRI adjustment for container: %s", nriErr.GetCMessage()); ++ NRIAdaptation::GetInstance()->UndoCreateContainer(sandbox, response_id, nriErr); ++ return response_id; ++ } ++#endif ++ + request = GenerateCreateContainerRequest(*sandbox, containerConfig, podSandboxConfig, error); + if (error.NotEmpty()) { + error.SetError("Failed to generate create container request"); + goto cleanup; + } ++#ifdef ENABLE_NRI ++ request->adjust = adjust; ++#endif + + if (m_cb->container.create(request, &response) != 0) { + if (response != nullptr && (response->errmsg != nullptr)) { +@@ -522,6 +540,12 @@ std::string ContainerManagerService::CreateContainer(const std::string &podSandb + + response_id = response->id; + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostCreateContainer(response_id, nriErr)) { ++ ERROR("NRI post-create notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif ++ + cleanup: + free_container_create_request(request); + free_container_create_response(response); +@@ -530,6 +554,9 @@ cleanup: + + void ContainerManagerService::StartContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + if (containerID.empty()) { + error.SetError("Invalid empty container id."); + return; +@@ -563,6 +590,14 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err + goto cleanup; + } + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StartContainer(containerID, nriErr)) { ++ ERROR("NRI container start failed: %s", nriErr.GetCMessage()); ++ NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr); ++ goto cleanup; ++ } ++#endif ++ + if (ret != 0) { + if (response != nullptr && response->errmsg != nullptr) { + error.SetError(response->errmsg); +@@ -571,6 +606,11 @@ void ContainerManagerService::StartContainer(const std::string &containerID, Err + } + goto cleanup; + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostStartContainer(containerID, nriErr)) { ++ ERROR("NRI PostStartContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + cleanup: + free_container_start_request(request); + free_container_start_response(response); +@@ -578,11 +618,25 @@ cleanup: + + void ContainerManagerService::StopContainer(const std::string &containerID, int64_t timeout, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + CRIHelpers::StopContainer(m_cb, containerID, timeout, error); ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->StopContainer(containerID, nriErr)) { ++ ERROR("NRI StopContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + void ContainerManagerService::RemoveContainer(const std::string &containerID, Errors &error) + { ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ if (!NRIAdaptation::GetInstance()->RemoveContainer(containerID, nriErr)) { ++ ERROR("NRI RemoveContainer notification failed: %s", nriErr.GetCMessage()); ++ } ++#endif + CRIHelpers::RemoveContainer(m_cb, containerID, error); + if (error.NotEmpty()) { + WARN("Failed to remove container %s", containerID.c_str()); +@@ -1048,6 +1102,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + struct parser_context ctx { + OPT_GEN_SIMPLIFY, 0 + }; ++ ++ runtime::v1::LinuxContainerResources updateRes = resources; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++ runtime::v1::LinuxContainerResources adjust; ++ if (!NRIAdaptation::GetInstance()->UpdateContainer(containerID, resources, adjust, nriErr)) { ++ ERROR("NRI UpdateContainer notification failed: %s", nriErr.GetCMessage()); ++ goto cleanup; ++ } ++ updateRes = adjust; ++#endif ++ + request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request)); + if (request == nullptr) { + error.SetError("Out of memory"); +@@ -1061,17 +1127,17 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + goto cleanup; + } + +- hostconfig->cpu_period = resources.cpu_period(); +- hostconfig->cpu_quota = resources.cpu_quota(); +- hostconfig->cpu_shares = resources.cpu_shares(); ++ hostconfig->cpu_period = updateRes.cpu_period(); ++ hostconfig->cpu_quota = updateRes.cpu_quota(); ++ hostconfig->cpu_shares = updateRes.cpu_shares(); + +- if (!resources.unified().empty()) { ++ if (!updateRes.unified().empty()) { + hostconfig->unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); + if (hostconfig->unified == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- for (auto &iter : resources.unified()) { ++ for (auto &iter : updateRes.unified()) { + if (append_json_map_string_string(hostconfig->unified, iter.first.c_str(), iter.second.c_str()) != 0) { + error.SetError("Failed to append string"); + goto cleanup; +@@ -1079,30 +1145,30 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + } + } + +- hostconfig->memory = resources.memory_limit_in_bytes(); +- hostconfig->memory_swap = resources.memory_swap_limit_in_bytes(); +- if (!resources.cpuset_cpus().empty()) { +- hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()); ++ hostconfig->memory = updateRes.memory_limit_in_bytes(); ++ hostconfig->memory_swap = updateRes.memory_swap_limit_in_bytes(); ++ if (!updateRes.cpuset_cpus().empty()) { ++ hostconfig->cpuset_cpus = util_strdup_s(updateRes.cpuset_cpus().c_str()); + } +- if (!resources.cpuset_mems().empty()) { +- hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); ++ if (!updateRes.cpuset_mems().empty()) { ++ hostconfig->cpuset_mems = util_strdup_s(updateRes.cpuset_mems().c_str()); + } +- if (resources.hugepage_limits_size() != 0) { ++ if (updateRes.hugepage_limits_size() != 0) { + hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( +- sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ sizeof(host_config_hugetlbs_element *), updateRes.hugepage_limits_size()); + if (hostconfig->hugetlbs == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ for (int i = 0; i < updateRes.hugepage_limits_size(); i++) { + hostconfig->hugetlbs[i] = + (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hostconfig->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + goto cleanup; + } +- hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); +- hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(updateRes.hugepage_limits(i).page_size().c_str()); ++ hostconfig->hugetlbs[i]->limit = updateRes.hugepage_limits(i).limit(); + hostconfig->hugetlbs_len++; + } + } +@@ -1121,6 +1187,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + error.SetError("Failed to call update container callback"); + } + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->PostUpdateContainer(containerID, nriErr)) { ++ ERROR("NRI PostUpdateContainer notification failed: %s", nriErr.GetCMessage()); ++ goto cleanup; ++ } ++#endif + cleanup: + free_container_update_request(request); + free_container_update_response(response); +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index 77faf48a..0140eb99 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -37,6 +37,9 @@ + #include "transform.h" + #include "isulad_config.h" + #include "mailbox.h" ++#ifdef ENABLE_NRI ++#include "nri_adaption.h" ++#endif + + namespace CRIV1 { + void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, +@@ -304,6 +307,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + std::string network_setting_json; + runtime::v1::PodSandboxConfig copyConfig = config; + cri_container_message_t msg = { 0 }; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + + // Step 1: Parepare sandbox name, runtime and networkMode + PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); +@@ -401,6 +407,14 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RunPodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ error.Errorf("NRI RunPodSandbox failed: %s", nriErr.GetCMessage()); ++ return response_id; ++ } ++#endif ++ + return sandbox->GetId(); + + cleanup_network: +@@ -418,6 +432,11 @@ cleanup_sandbox: + if (error.NotEmpty()) { + ERROR("Failed to delete sandbox: %s", sandbox->GetId().c_str()); + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { ++ DEBUG("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ } ++#endif + + return response_id; + } +@@ -618,7 +637,15 @@ void PodSandboxManagerService::StopPodSandbox(const std::string &podSandboxID, E + return; + } + +- sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error); ++ if (!sandbox->Stop(sandbox::DEFAULT_STOP_TIMEOUT, error)) { ++ return; ++ } ++#ifdef ENABLE_NRI ++ Errors nriStopPodErr; ++ if (!NRIAdaptation::GetInstance()->StopPodSandbox(sandbox, nriStopPodErr)) { ++ ERROR("NRI sandbox stop notification failed %s", nriStopPodErr.GetCMessage()); ++ } ++#endif + } + + void PodSandboxManagerService::RemoveAllContainersInSandbox(const std::string &readSandboxID, +@@ -656,6 +683,9 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + { + std::vector errors; + std::string realSandboxID; ++#ifdef ENABLE_NRI ++ Errors nriErr; ++#endif + + if (m_cb == nullptr || m_cb->container.remove == nullptr) { + ERROR("Unimplemented callback"); +@@ -724,6 +754,11 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; + mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); + } ++#ifdef ENABLE_NRI ++ if (!NRIAdaptation::GetInstance()->RemovePodSandbox(sandbox, nriErr)) { ++ ERROR("NRI RemovePodSandbox failed: %s", nriErr.GetCMessage()); ++ } ++#endif + } + + auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 2b6b2a15..dcbdd1d3 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -65,6 +65,10 @@ + #include "mailbox.h" + #include "specs_mount.h" + ++#ifdef ENABLE_NRI ++#include "nri_spec.h" ++#endif ++ + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) + { +@@ -1465,6 +1469,14 @@ int container_create_cb(const container_create_request *request, container_creat + skip_sandbox_key_manage = (is_sandbox_container(request->sandbox) && namespace_is_cni(host_spec->network_mode)); + #endif + ++#ifdef ENABLE_NRI ++ if (request->adjust != NULL && nri_adjust_host_spec(request->adjust, host_spec) != 0) { ++ ERROR("Failed to adjust host spec"); ++ cc = ISULAD_ERR_INPUT; ++ goto clean_container_root_dir; ++ } ++#endif ++ + if (save_container_config_before_create(id, runtime_root, host_spec, v2_spec) != 0) { + ERROR("Failed to malloc container_config_v2_common_config"); + cc = ISULAD_ERR_INPUT; +@@ -1553,6 +1565,15 @@ int container_create_cb(const container_create_request *request, container_creat + } + #endif + ++#ifdef ENABLE_NRI ++ // modify oci spec by nri plugin ++ if (request->adjust != NULL && nri_adjust_oci_spec(request->adjust, oci_spec) != 0) { ++ ERROR("Failed to adjust oci spec"); ++ cc = ISULAD_ERR_EXEC; ++ goto clean_netns; ++ } ++#endif ++ + host_channel = dup_host_channel(host_spec->host_channel); + if (prepare_host_channel(host_channel, host_spec->user_remap)) { + ERROR("Failed to prepare host channel"); +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index 0b95cdad..4157c631 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -76,6 +76,9 @@ + #include "sandbox_ops.h" + #include "vsock_io_handler.h" + #endif ++#ifdef ENABLE_NRI ++#include "nri_spec.h" ++#endif + + #define KATA_RUNTIME "kata-runtime" + +@@ -726,6 +729,15 @@ static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, contai + return -1; + } + ++#ifdef ENABLE_NRI ++ // update oci spec with nri ++ ret = update_oci_nri(oci_spec, hostconfig); ++ if (ret != 0) { ++ ERROR("Failed to update oci spec with nri"); ++ return -1; ++ } ++#endif ++ + // renew_oci_config() will update process->user and share namespace after. + + return 0; +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 002431d8..36e89343 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -2883,7 +2883,12 @@ int spec_add_linux_resources_rlimit(oci_runtime_spec *oci_spec, const char *type + return -1; + } + +- ret = make_sure_oci_spec_linux_resources(oci_spec); ++ ret = make_sure_oci_spec_process(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ ret = merge_ulimits_pre(oci_spec, 1); + if (ret < 0) { + return -1; + } +diff --git a/src/daemon/nri/nri_adaption.cc b/src/daemon/nri/nri_adaption.cc +new file mode 100644 +index 00000000..3190767f +--- /dev/null ++++ b/src/daemon/nri/nri_adaption.cc +@@ -0,0 +1,1165 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin manager(NRI adaption) class definition ++ *********************************************************************************/ ++ ++#include "nri_adaption.h" ++ ++#include ++ ++#include ++#include ++ ++#include "isulad_config.h" ++#include "utils_file.h" ++#include "utils_string.h" ++#include "utils.h" ++#include "nri_convert.h" ++#include "nri_plugin.h" ++#include "nri_result.h" ++#include "sandbox_manager.h" ++ ++std::atomic NRIAdaptation::m_instance; ++ ++NRIAdaptation *NRIAdaptation::GetInstance() noexcept ++{ ++ static std::once_flag flag; ++ ++ std::call_once(flag, [] { m_instance = new NRIAdaptation; }); ++ ++ return m_instance; ++} ++ ++NRIAdaptation::~NRIAdaptation() ++{ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ plugin->shutdown(); ++ } ++} ++ ++auto NRIAdaptation::Init(Errors &error) -> bool ++{ ++ std::map> tmp_storeMap; ++ ++ m_support = conf_get_nri_support(); ++ if (!m_support) { ++ return true; ++ } ++ ++ m_external_support = conf_get_nri_external_support(); ++ service_executor_t *cb = get_service_executor(); ++ if (cb == nullptr) { ++ ERROR("Init isulad service executor failure."); ++ return false; ++ } ++ ++ m_containerManager = std::make_unique(cb); ++ ++ m_pluginConfigPath = GetNRIPluginConfigPath(); ++ m_pluginPath = GetNRIPluginPath(); ++ m_plugin_registration_timeout = conf_get_nri_plugin_registration_timeout(); ++ m_plugin_requst_timeout = conf_get_nri_plugin_requst_timeout(); ++ m_sock_path = GetNRISockPath(); ++ ++ if (!StartPlugin()) { ++ ERROR("Failed to do StartPlugin"); ++ return false; ++ } ++ ++ if (!SortPlugins()) { ++ ERROR("Failed to do SortPlugins"); ++ return false; ++ } ++ ++ return true; ++} ++ ++void NRIAdaptation::RemoveClosedPlugins() ++{ ++ std::vector closedPlugin; ++ ++ GetClosedPlugins(closedPlugin); ++ ++ for (const auto &key : closedPlugin) { ++ RemovePluginByIndex(key); ++ } ++} ++ ++void NRIAdaptation::GetClosedPlugins(std::vector &closedPlugin) ++{ ++ ReadGuard lock(m_mutex); ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ if (plugin->IsClose()) { ++ closedPlugin.push_back(pair.first); ++ } ++ } ++} ++ ++auto NRIAdaptation::IsSupport() -> bool ++{ ++ return m_support; ++} ++ ++auto NRIAdaptation::GetPluginByIndex(const std::string &index) -> std::shared_ptr ++{ ++ ReadGuard lock(m_mutex); ++ return m_storeMap[index]; ++} ++ ++void NRIAdaptation::RemovePluginByIndex(const std::string &index) ++{ ++ WriteGuard lock(m_mutex); ++ m_storeMap.erase(index); ++} ++ ++void NRIAdaptation::AddPluginByIndex(const std::string &index, std::shared_ptr plugin) ++{ ++ WriteGuard lock(m_mutex); ++ m_storeMap[index] = plugin; ++} ++ ++auto NRIAdaptation::ApplyUpdates(const std::vector &update, ++ std::vector &failed, bool getFailed, Errors &error) -> bool ++{ ++ for (auto &u : update) { ++ runtime::v1::LinuxContainerResources resources; ++ if (!LinuxResourcesFromNRI(u->linux->resources, resources)) { ++ ERROR("Failed to convert Linux resources from NRI"); ++ error.Errorf("Failed to convert Linux resources from NRI"); ++ return false; ++ } ++ ++ m_containerManager->UpdateContainerResources(u->container_id, resources, error); ++ ++ if (error.NotEmpty()) { ++ ERROR("Failed to update container: %s resources: %s", u->container_id, error.GetCMessage()); ++ if (!u->ignore_failure && getFailed) { ++ failed.push_back(u); ++ } ++ continue; ++ } ++ ++ TRACE("NRI update of container %s successful", u->container_id); ++ } ++ ++ if (failed.size() != 0) { ++ error.Errorf("NRI update of some containers failed"); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIAdaptation::NewExternalPlugin(int fd) -> bool ++{ ++ if (!m_external_support) { ++ ERROR("External plugin support is disabled"); ++ return false; ++ } ++ if (fd < 0) { ++ ERROR("Invalid fd"); ++ return false; ++ } ++ ++ std::string plugin_name; ++ NRIHelpers::GenerateRandomExternalName(plugin_name); ++ if (plugin_name.empty()) { ++ ERROR("Failed to generate random external name"); ++ return false; ++ } ++ ++ auto plugin = std::make_shared(fd, plugin_name); ++ ++ AddPluginByIndex(plugin_name, plugin); ++ ++ if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { ++ ERROR("Failed to start plugin ready for conn fd %d", fd); ++ RemovePluginByIndex(plugin_name); ++ return false; ++ } ++ ++ std::vector pods; ++ std::vector cons; ++ nri_container_update **updateRes; ++ size_t update_len = 0; ++ ++ std::vector> sandboxes; ++ runtime::v1::PodSandboxFilter podFilter; ++ std::vector> containers; ++ Errors tmpError; ++ ++ std::vector updates; ++ std::vector failed; ++ ++ sandbox::SandboxManager::GetInstance()->ListAllSandboxes(podFilter, sandboxes); ++ ++ if (!PodSandboxesToNRI(sandboxes, pods)) { ++ ERROR("Failed to convert podsandbox to nri"); ++ NRIHelpers::FreeNriPodVector(pods); ++ return false; ++ } ++ ++ m_containerManager->ListContainers(nullptr, containers, tmpError); ++ ++ if (!ContainersToNRI(containers, cons)) { ++ ERROR("Failed to convert container to nri"); ++ NRIHelpers::FreeNriPodVector(pods); ++ NRIHelpers::FreeNriContainerVector(cons); ++ return false; ++ } ++ ++ // pods and cons's memory transfer to nri_synchronize_request, ++ // and is automatically freed when nri_synchronize_request is freed ++ if (!plugin->Synchronize(pods, cons, &updateRes, update_len, tmpError)) { ++ ERROR("Failed to synchronize plugin"); ++ return false; ++ } ++ ++ for (size_t i = 0; i < update_len; i++) { ++ updates.push_back(updateRes[i]); ++ } ++ ++ if (!ApplyUpdates(updates, failed, false, tmpError)) { ++ ERROR("Failed to update post-sync"); ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(updates); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ INFO("plugin %s connected", plugin_name.c_str()); ++ return true; ++} ++ ++auto NRIAdaptation::RunPodSandbox(std::shared_ptr sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto runPodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (runPodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ runPodEvent->get()->pod = pod->move(); ++ runPodEvent->get()->event = RUN_POD_SANDBOX; ++ return StateChange(runPodEvent->get(), error); ++} ++ ++auto NRIAdaptation::StopPodSandbox(std::shared_ptr sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto stopPodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (stopPodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ stopPodEvent->get()->pod = pod->move(); ++ stopPodEvent->get()->event = STOP_POD_SANDBOX; ++ return StateChange(stopPodEvent->get(), error); ++} ++ ++auto NRIAdaptation::RemovePodSandbox(std::shared_ptr sandbox, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto removePodEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (removePodEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ removePodEvent->get()->pod = pod->move(); ++ removePodEvent->get()->event = REMOVE_POD_SANDBOX; ++ return StateChange(removePodEvent->get(), error); ++} ++ ++auto NRIAdaptation::CreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ const runtime::v1::ContainerConfig &containerConfig, nri_container_adjustment **adjust, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto con = NRIContainerByConConfig(sandbox, containerConfig, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_create_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->container = con->move(); ++ req->get()->pod = pod->move(); ++ ++ pluginResult result; ++ result.InitByConId(conId); ++ ++ if (!PluginsCreateContainer(req->get(), conId, result)) { ++ ERROR("Failed to call create container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:evict container do not aply ++ ++ // TODO:how can i rollback on failure ++ std::vector failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ *adjust = result.MoveReplyAdjust(); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return true; ++} ++ ++bool NRIAdaptation::PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_create_container_response *resp = nullptr; ++ ++ if (!plugin->CreateContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ (void)plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty CreateContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = ++ makeUniquePtrCStructWrapper(resp, free_nri_create_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(CREATE_CONTAINER, resp_wrapper->get()->adjust, resp_wrapper->get()->update, ++ resp_wrapper->get()->update_len, plugin->GetName()); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::PostCreateContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postCreateConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (postCreateConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postCreateConEvent->get()->container = con->move(); ++ postCreateConEvent->get()->pod = pod->move(); ++ postCreateConEvent->get()->event = POST_CREATE_CONTAINER; ++ return StateChange(postCreateConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StartContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox for container: %s, sandbox id: %s", conId.c_str(), con->get()->pod_sandbox_id); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto startConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (startConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ startConEvent->get()->container = con->move(); ++ startConEvent->get()->pod = pod->move(); ++ startConEvent->get()->event = START_CONTAINER; ++ return StateChange(startConEvent->get(), error); ++} ++ ++auto NRIAdaptation::PostStartContainer(const std::string &conId, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postStartConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (postStartConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postStartConEvent->get()->container = con->move(); ++ postStartConEvent->get()->pod = pod->move(); ++ postStartConEvent->get()->event = POST_START_CONTAINER; ++ return StateChange(postStartConEvent->get(), error); ++} ++ ++auto NRIAdaptation::UndoCreateContainer(std::shared_ptr sandbox, const std::string &conId, ++ Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ if (!StopContainer(conId, error)) { ++ ERROR("container creation undo (stop) failed: %s", conId.c_str()); ++ } ++ ++ if (!RemoveContainer(conId, error)) { ++ ERROR("container creation undo (remove) failed: %s", conId.c_str()); ++ } ++ ++ return true; ++} ++ ++auto NRIAdaptation::UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, ++ runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_update_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ auto reqRes = LinuxResourcesToNRI(resources); ++ ++ req->get()->container = con->move(); ++ req->get()->pod = pod->move(); ++ req->get()->linux_resources = reqRes; ++ ++ pluginResult result; ++ result.InitByUpdateReq(req->get()); ++ ++ if (!PluginsUpdateContainer(req->get(), conId, result)) { ++ ERROR("Failed to call update container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:evict container do not aply ++ ++ // TODO:how can i rollback on failure ++ std::vector failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ ++ if (!LinuxResourcesFromNRI(result.GetReplyResources(conId), adjust)) { ++ ERROR("Failed to convert Linux resources from NRI"); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool NRIAdaptation::PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_update_container_response *resp = nullptr; ++ ++ if (!plugin->UpdateContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty UpdateContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = ++ makeUniquePtrCStructWrapper(resp, free_nri_update_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(UPDATE_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, ++ plugin->GetName()); ++ } ++ return true; ++} ++ ++bool NRIAdaptation::PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, ++ pluginResult &result) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ nri_stop_container_response *resp = nullptr; ++ ++ if (!plugin->StopContainer(req, &resp, tmpError)) { ++ ERROR("Failed to call create container: %s to pliugin: %s", conId.c_str(), plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ ++ if (resp == nullptr) { ++ ERROR("Empty StopContainer resp : %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_stop_container_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ result.Apply(STOP_CONTAINER, nullptr, resp_wrapper->get()->update, resp_wrapper->get()->update_len, plugin->GetName()); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::PostUpdateContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto postUpdateConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (postUpdateConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ postUpdateConEvent->get()->container = con->move(); ++ postUpdateConEvent->get()->pod = pod->move(); ++ postUpdateConEvent->get()->event = POST_UPDATE_CONTAINER; ++ return StateChange(postUpdateConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StopContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_stop_container_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->pod = pod->move(); ++ req->get()->container = con->move(); ++ ++ pluginResult result; ++ result.Init(); ++ ++ if (!PluginsStopContainer(req->get(), conId, result)) { ++ ERROR("Failed to call stop container to all plugins"); ++ return false; ++ } ++ ++ RemoveClosedPlugins(); ++ ++ // TODO:how can i rollback on failure ++ std::vector failed; ++ if (!ApplyUpdates(result.GetReplyUpdate(), failed, false, error)) { ++ ERROR("Failed to apply updates"); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return false; ++ } ++ ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return true; ++} ++ ++auto NRIAdaptation::RemoveContainer(const std::string &conId, Errors &error) ->bool ++{ ++ if (!m_support) { ++ return true; ++ } ++ ++ auto con = NRIContainerByID(conId, error); ++ if (con == nullptr) { ++ ERROR("Failed to covert container to nri: %s", conId.c_str()); ++ return false; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(con->get()->pod_sandbox_id); ++ if (sandbox == nullptr) { ++ ERROR("Failed to get sandbox info for nri"); ++ return false; ++ } ++ ++ auto pod = NRIPodSandbox(sandbox, error); ++ if (pod == nullptr) { ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return false; ++ } ++ ++ auto removeConEvent = makeUniquePtrCStructWrapper(free_nri_state_change_event); ++ if (removeConEvent == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ removeConEvent->get()->container = con->move(); ++ removeConEvent->get()->pod = pod->move(); ++ removeConEvent->get()->event = REMOVE_CONTAINER; ++ return StateChange(removeConEvent->get(), error); ++} ++ ++auto NRIAdaptation::StateChange(nri_state_change_event *evt, Errors &error) ->bool ++{ ++ if (evt->event == UNKNOWN) { ++ ERROR("invalid (unset) event in state change notification"); ++ error.SetError("invalid (unset) event in state change notification"); ++ return false; ++ } ++ ++ PluginsStateChange(evt); ++ ++ RemoveClosedPlugins(); ++ return true; ++} ++ ++void NRIAdaptation::PluginsStateChange(nri_state_change_event *evt) ++{ ++ ReadGuard lock(m_mutex); ++ ++ for (const auto &pair : m_storeMap) { ++ auto plugin = pair.second; ++ Errors tmpError; ++ if (!plugin->StateChange(evt, tmpError)) { ++ ERROR("invalid (unset) event in state change notification: %s", plugin->GetName().c_str()); ++ plugin->shutdown(); ++ continue; ++ } ++ } ++} ++ ++// Perform a set of unsolicited container updates requested by a plugin. ++auto NRIAdaptation::updateContainers(const nri_update_containers_request *req, ++ nri_update_containers_response **resp) ->bool ++{ ++ std::vector failed; ++ std::vector vec; ++ size_t i; ++ Errors error; ++ bool ret = false; ++ ++ if (req == nullptr) { ++ ERROR("Invalid request"); ++ return false; ++ } ++ ++ for (i = 0; i < req->update_len; i++) { ++ vec.push_back(req->update[i]); ++ } ++ ++ if (!ApplyUpdates(vec, failed, false, error)) { ++ ERROR("Failed to apply updates: %s", error.GetCMessage()); ++ goto free_out; ++ } ++ ++ if (failed.size() == 0) { ++ ret = true; ++ goto free_out; ++ } ++ ++ (*resp)->failed = (nri_container_update **)util_common_calloc_s(failed.size() * sizeof(nri_container_update *)); ++ if ((*resp)->failed == nullptr) { ++ ERROR("Out of memory"); ++ goto free_out; ++ } ++ ++ for (i = 0; i < failed.size(); i++) { ++ (*resp)->failed[i] = failed[i]; ++ failed[i] = nullptr; ++ (*resp)->failed_len++; ++ } ++ ret = true; ++ ++free_out: ++ NRIHelpers::FreeNriContainerUpdateVector(vec); ++ NRIHelpers::FreeNriContainerUpdateVector(failed); ++ return ret; ++} ++ ++auto NRIAdaptation::StartPlugin() -> bool ++{ ++ std::map> tmp_storeMap; ++ ++ if (!DiscoverPlugins(tmp_storeMap) != 0) { ++ ERROR("Failed to do DiscoverPlugins"); ++ return false; ++ } ++ ++ for (const auto &pair : tmp_storeMap) { ++ const std::string &index = pair.first; ++ auto plugin = pair.second; ++ ++ if (!NewLaunchedPlugin(plugin)) { ++ ERROR("Failed to do NewLaunchedPlugin for %s", plugin->GetName().c_str()); ++ continue; ++ } ++ ++ AddPluginByIndex(index, plugin); ++ ++ if (!plugin->Start(m_plugin_registration_timeout, m_plugin_requst_timeout)) { ++ ERROR("Failed to start plugin %s ready", plugin->GetName().c_str()); ++ RemovePluginByIndex(index); ++ (void)plugin->shutdown(); ++ continue; ++ } ++ } ++ ++ return true; ++} ++ ++// return true always, failure to obtain one plugin does not affect other plugin ++static auto walk_plugin_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) -> bool ++{ ++ std::string full_path = std::string(path_name) + "/" + sub_dir->d_name; ++ struct stat file_stat = { 0 }; ++ ++ if (stat(full_path.c_str(), &file_stat) != 0) { ++ WARN("Failed to get NRI plugin %s stat", sub_dir->d_name); ++ return true; ++ } ++ ++ if (S_ISDIR(file_stat.st_mode)) { ++ INFO("Skip dir in plugin path %s", sub_dir->d_name); ++ return true; ++ } ++ ++ // 1. Verify plugin permissions for exec ++ if (!(file_stat.st_mode & S_IXUSR)) { ++ ERROR("NRI plugin %s has no permission for exec", sub_dir->d_name); ++ return true; ++ } ++ ++ // 2. Parse plugin name ++ __isula_auto_array_t char **arr = util_string_split_n(sub_dir->d_name, '-', 2); ++ if (arr == nullptr) { ++ ERROR("Invalid plugin name %s, idx-pluginname expected", sub_dir->d_name); ++ return true; ++ } ++ ++ if (!NRIHelpers::CheckPluginIndex(arr[0])) { ++ ERROR("Invalid plugin name %s, invalid idx", sub_dir->d_name); ++ return true; ++ } ++ ++ // 3. init plugin ++ std::string index(arr[0]); ++ std::string pluginName(arr[1]); ++ std::string config; ++ ++ std::map> &map = ++ *static_cast>*>(context); ++ if (!NRIHelpers::GetPluginConfig(index, pluginName, config)) { ++ ERROR("Failed to get plugin %s config", pluginName.c_str()); ++ return true; ++ } ++ ++ auto plugin = std::make_shared(index, pluginName, config); ++ ++ // todo:use random id str ++ map[pluginName] = plugin; ++ return true; ++} ++ ++static void plugin_exec_func(nri_plugin_exec_args_t * plugin_args) ++{ ++ const char *params[PARAM_NUM] = {0}; ++ int i = 0; ++ std::string sock = std::to_string(plugin_args->sockFd); ++ ++ if (plugin_args == nullptr) { ++ ERROR("Missing plugin exec info"); ++ _exit(EXIT_FAILURE); ++ } ++ ++ if (chdir(plugin_args->workdir) < 0) { ++ ERROR("Failed to chdir to %s", plugin_args->workdir); ++ _exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginNameEnv.c_str(), plugin_args->name, 1) != 0) { ++ ERROR("%s: failed to set PluginNameEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginIdxEnv.c_str(), plugin_args->index, 1) != 0) { ++ ERROR("%s: failed to set PluginIdxEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setenv(PluginSocketEnv.c_str(), sock.c_str(), 1) != 0) { ++ ERROR("%s: failed to set PluginSocketEnv for process %d", plugin_args->name, getpid()); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (util_check_inherited(true, plugin_args->sockFd) != 0) { ++ ERROR("Failed to close inherited fds"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (setsid() < 0) { ++ ERROR("Failed to setsid for nri plugin: %s", plugin_args->name); ++ exit(EXIT_FAILURE); ++ } ++ ++ params[i++] = plugin_args->name; ++ ++ execvp(plugin_args->cmd, (char * const *)params); ++ ERROR("Failed to exec %s", plugin_args->cmd); ++ _exit(EXIT_FAILURE); ++} ++ ++// create socket, and call plugin start ++auto NRIAdaptation::NewLaunchedPlugin(const std::shared_ptr &plugin) -> bool ++{ ++ // 1. create socket for plugin ++ if (!plugin->CreateSocketPair()) { ++ ERROR("Failed to create socket pair"); ++ return false; ++ } ++ ++ std::string name = plugin->GetQualifiedName(); ++ std::string cmd = m_pluginPath + "/" + name; ++ ++ DEBUG("Plugin %s start", cmd.c_str()); ++ ++ // 2. exec plugin ++ nri_plugin_exec_args_t p_args = { ++ .workdir = m_pluginPath.c_str(), ++ .cmd = cmd.c_str(), ++ .name = name.c_str(), ++ .index = plugin->GetIndex().c_str(), ++ .sockFd = plugin->GetPeerSockFd(), ++ }; ++ ++ int pid = fork(); ++ if (pid == (pid_t) -1) { ++ SYSERROR("Failed to fork"); ++ return false; ++ } ++ ++ if (pid == (pid_t)0) { ++ set_child_process_pdeathsig(); ++ ++ plugin_exec_func(&p_args); ++ } ++ ++ close(plugin->GetPeerSockFd()); ++ ++ plugin->SetPid(pid); ++ ++ return true; ++} ++ ++// find plugin and create plugin ++auto NRIAdaptation::DiscoverPlugins(std::map> &map) -> bool ++{ ++ int nret = 0; ++ ++ // 1. get all plugin ++ nret = util_scan_subdirs(m_pluginPath.c_str(), walk_plugin_dir_cb, static_cast(&map)); ++ if (nret != 0) { ++ ERROR("Failed to scan nri plugin subdirs"); ++ } ++ return true; ++} ++ ++auto NRIAdaptation::SortPlugins() -> bool ++{ ++ RemoveClosedPlugins(); ++ ++ std::vector>> sortedPlugins(m_storeMap.begin(), m_storeMap.end()); ++ ++ std::sort(sortedPlugins.begin(), sortedPlugins.end(), [](const auto & a, const auto & b) { ++ return a.first < b.first; ++ }); ++ ++ WriteGuard lock(m_mutex); ++ m_storeMap.clear(); ++ for (const auto &pair : sortedPlugins) { ++ m_storeMap.insert(pair); ++ } ++ ++ return true; ++} ++ ++auto NRIAdaptation::GetNRIPluginConfigPath(void) -> std::string ++{ ++ __isula_auto_free char *config_path = nullptr; ++ std::string ret; ++ ++ config_path = conf_get_nri_plugin_config_path(); ++ if (config_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(config_path); ++ return ret; ++} ++ ++auto NRIAdaptation::GetNRIPluginPath(void) -> std::string ++{ ++ __isula_auto_free char *plugin_path = nullptr; ++ std::string ret; ++ ++ plugin_path = conf_get_nri_plugin_path(); ++ if (plugin_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(plugin_path); ++ return ret; ++} ++ ++auto NRIAdaptation::GetNRISockPath(void) -> std::string ++{ ++ __isula_auto_free char *sock_path = nullptr; ++ std::string ret; ++ ++ sock_path = conf_get_socket_path(); ++ if (sock_path == nullptr) { ++ return ret; ++ } ++ ret = std::string(sock_path); ++ return ret; ++} ++ ++auto NRIAdaptation::NRIPodSandbox(const std::shared_ptr &sandbox, ++ Errors &error) -> std::unique_ptr> ++{ ++ auto pod = makeUniquePtrCStructWrapper(free_nri_pod_sandbox); ++ if (pod == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!PodSandboxToNRI(sandbox, *pod->get())) { ++ error.Errorf("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ ERROR("Failed to covert podsandbox to nri: %s", sandbox->GetId().c_str()); ++ return nullptr; ++ } ++ ++ return pod; ++} ++ ++auto NRIAdaptation::NRIContainerByID(const std::string &id, ++ Errors &error) -> std::unique_ptr> ++{ ++ auto con = makeUniquePtrCStructWrapper(free_nri_container); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!ContainerToNRIByID(id, *con->get())) { ++ error.Errorf("Failed to covert container to nri: %s", id.c_str()); ++ ERROR("Failed to covert container to nri: %s", id.c_str()); ++ return nullptr; ++ } ++ ++ return con; ++} ++ ++auto NRIAdaptation::NRIContainerByConConfig(const std::shared_ptr &sandbox, ++ const runtime::v1::ContainerConfig &containerConfig, Errors &error) -> std::unique_ptr> ++{ ++ auto con = makeUniquePtrCStructWrapper(free_nri_container); ++ if (con == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ if (!ContainerToNRIByConConfig(containerConfig, *con->get())) { ++ error.Errorf("Failed to covert container to nri: %s", con->get()->name); ++ ERROR("Failed to covert container to nri: %s", con->get()->name); ++ return nullptr; ++ } ++ con->get()->pod_sandbox_id = isula_strdup_s(sandbox->GetId().c_str()); ++ ++ return con; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h +index 874662cf..27a6d93e 100644 +--- a/src/daemon/nri/nri_adaption.h ++++ b/src/daemon/nri/nri_adaption.h +@@ -46,10 +46,6 @@ public: + + auto GetSockpath(std::vector &paths) -> bool; + +- auto StopPlugins() -> bool; +- +- void RemoveClosedPlugins(); +- + auto GetPluginByIndex(const std::string &index) -> std::shared_ptr; + void AddPluginByIndex(const std::string &index, std::shared_ptr plugin); + void RemovePluginByIndex(const std::string &index); +@@ -65,7 +61,8 @@ public: + Errors &error) -> bool; + auto StartContainer(const std::string &conId, Errors &error) -> bool; + auto PostStartContainer(const std::string &conId, Errors &error) -> bool; +- auto UpdateContainer(const std::string &conId, Errors &error) -> bool; ++ auto UpdateContainer(const std::string &conId, const runtime::v1::LinuxContainerResources &resources, ++ runtime::v1::LinuxContainerResources &adjust, Errors &error) -> bool; + auto PostUpdateContainer(const std::string &conId, Errors &error) -> bool; + auto StopContainer(const std::string &conId, Errors &error) -> bool; + auto RemoveContainer(const std::string &conId, Errors &error) -> bool; +@@ -87,6 +84,8 @@ private: + auto SyncPlugin() -> bool; + + auto SortPlugins() -> bool; ++ void RemoveClosedPlugins(); ++ + void GetClosedPlugins(std::vector &closedPlugin); + + auto ApplyUpdates(const std::vector &update, std::vector &failed, +@@ -107,18 +106,16 @@ private: + void PluginsStateChange(nri_state_change_event *evt); + bool PluginsCreateContainer(nri_create_container_request *req, const std::string &conId, pluginResult &result); + bool PluginsUpdateContainer(nri_update_container_request *req, const std::string &conId, pluginResult &result); ++ bool PluginsStopContainer(nri_stop_container_request *req, const std::string &conId, pluginResult &result); + + private: + RWMutex m_mutex; + static std::atomic m_instance; + bool m_support; + bool m_external_support; +- std::string m_version; + std::string m_sock_path; + std::string m_pluginConfigPath; + std::string m_pluginPath; +- std::vector m_socketPathArr; +- std::string m_disableConnections; + // id --> NRIPlugin map + std::map> m_storeMap; + // TODO:plugin monitor thread id?? +diff --git a/src/daemon/nri/nri_helpers.cc b/src/daemon/nri/nri_helpers.cc +index ff9d67c1..b660e7a7 100644 +--- a/src/daemon/nri/nri_helpers.cc ++++ b/src/daemon/nri/nri_helpers.cc +@@ -90,4 +90,25 @@ bool CheckPluginIndex(const std::string &idx) + + return true; + } ++ ++void FreeNriContainerUpdateVector(std::vector &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_container_update(ptr); ++ } ++} ++ ++void FreeNriContainerVector(std::vector &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_container(ptr); ++ } ++} ++ ++void FreeNriPodVector(std::vector &vec) ++{ ++ for (auto ptr : vec) { ++ free_nri_pod_sandbox(ptr); ++ } ++} + }// namespace NRIHelpers +\ No newline at end of file +diff --git a/src/daemon/nri/nri_helpers.h b/src/daemon/nri/nri_helpers.h +index 1a2f488e..e776987b 100644 +--- a/src/daemon/nri/nri_helpers.h ++++ b/src/daemon/nri/nri_helpers.h +@@ -41,6 +41,10 @@ void GenerateRandomExternalName(std::string &ret); + + bool CheckPluginIndex(const std::string &idx); + ++void FreeNriContainerUpdateVector(std::vector &vec); ++void FreeNriContainerVector(std::vector &vec); ++void FreeNriPodVector(std::vector &vec); ++ + template + void freeArray(T ** &arr, int size) + { +diff --git a/src/daemon/nri/nri_plugin_ops.cc b/src/daemon/nri/nri_plugin_ops.cc +new file mode 100644 +index 00000000..e2f88b63 +--- /dev/null ++++ b/src/daemon/nri/nri_plugin_ops.cc +@@ -0,0 +1,123 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-26 ++ * Description: provide nri plugin api definition ++ ******************************************************************************/ ++#include "nri_plugin_ops.h" ++ ++#include ++#include ++ ++#include "nri_adaption.h" ++#include "nri_plugin.h" ++#include "isulad_config.h" ++ ++static bool start_external_listener() ++{ ++ __isula_auto_free char *sock_path = NULL; ++ ++ sock_path = conf_get_socket_path(); ++ if (sock_path == NULL) { ++ ERROR("Failed to get socket path"); ++ return false; ++ } ++ ++ if (nri_external_service_start(sock_path, nri_external_plugin_connect) != 0) { ++ ERROR("Failed to lauch external service"); ++ return false; ++ } ++ return true; ++} ++ ++bool nri_adaption_init(void) ++{ ++ Errors error; ++ ++ if (!conf_get_nri_support()) { ++ return true; ++ } ++ ++ nri_runtime_callbacks callbacks; ++ callbacks.register_plugin = nri_registry_containers; ++ callbacks.update_containers = nri_update_containers; ++ if (nri_runtime_service_init(callbacks) != 0) { ++ ERROR("Failed to init runtime service\n"); ++ return false; ++ } ++ ++ if (conf_get_nri_external_support()) { ++ if (!start_external_listener()) { ++ ERROR("Failed to start external listener\n"); ++ goto clean_out; ++ } ++ } ++ ++ NRIAdaptation::GetInstance()->Init(error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to init NRIAdaptation: %s", error.GetCMessage()); ++ goto clean_out; ++ } ++ return true; ++clean_out: ++ nri_runtime_service_destroy(); ++ return false; ++} ++ ++bool nri_adaption_shutdown(void) ++{ ++ nri_external_service_shutdown(); ++ nri_runtime_service_destroy(); ++ return true; ++} ++ ++int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, ++ nri_update_containers_response **response) ++{ ++ if (request == nullptr || response == nullptr || plugin_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ if (!NRIAdaptation::GetInstance()->updateContainers(request, response)) { ++ ERROR("Failed to update containers by plugin %s", plugin_id); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request) ++{ ++ if (request == nullptr || plugin_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ auto plugin = NRIAdaptation::GetInstance()->GetPluginByIndex(plugin_id); ++ if (plugin == nullptr) { ++ ERROR("Failed to get plugin by index %s", plugin_id); ++ return -1; ++ } ++ ++ plugin->SetReady(); ++ return 0; ++} ++ ++int nri_external_plugin_connect(int fd) ++{ ++ if (fd < 0) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ return NRIAdaptation::GetInstance()->NewExternalPlugin(fd) ? 0 : -1; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_plugin_ops.h b/src/daemon/nri/nri_plugin_ops.h +index 37d437d2..3a5393ba 100644 +--- a/src/daemon/nri/nri_plugin_ops.h ++++ b/src/daemon/nri/nri_plugin_ops.h +@@ -25,13 +25,15 @@ extern "C" { + #endif + + bool nri_adaption_init(void); ++bool nri_adaption_shutdown(void); + + #ifdef __cplusplus + } + #endif + +-int nri_update_containers(const nri_update_containers_request *request, nri_update_containers_response **response); +-int nri_registry_containers(const nri_register_plugin_request *request); ++int nri_update_containers(const char *plugin_id, const nri_update_containers_request *request, ++ nri_update_containers_response **response); ++int nri_registry_containers(const char *plugin_id, const nri_register_plugin_request *request); + + int nri_external_plugin_connect(int fd); + +diff --git a/src/daemon/nri/nri_result.cc b/src/daemon/nri/nri_result.cc +new file mode 100644 +index 00000000..7da4451d +--- /dev/null ++++ b/src/daemon/nri/nri_result.cc +@@ -0,0 +1,977 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-06-29 ++ * Description: provide nri result definition ++ *********************************************************************************/ ++ ++#include "nri_result.h" ++ ++#include ++#include ++ ++#include "cxxutils.h" ++#include "transform.h" ++#include "utils.h" ++ ++pluginResult::~pluginResult() ++{ ++ free_nri_linux_resources(m_update_req); ++ free_nri_container_adjustment(m_reply.adjust); ++ for (size_t i = 0; i < m_reply.update.size(); i++) { ++ free_nri_container_update(m_reply.update[i]); ++ } ++} ++ ++auto pluginResult::InitReply() -> bool ++{ ++ m_reply.adjust = (nri_container_adjustment *)util_common_calloc_s(sizeof(nri_container_adjustment)); ++ if (m_reply.adjust == NULL) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ return true; ++} ++ ++auto pluginResult::Init() -> bool ++{ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::InitByConId(std::string conId) -> bool ++{ ++ m_conId = conId; ++ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::InitByUpdateReq(nri_update_container_request *req) -> bool ++{ ++ m_conId = req->container->id; ++ m_update_req = copy_nri_linux_resources(req->linux_resources); ++ if (m_update_req == nullptr) { ++ ERROR("Failed to copy nri linux resources"); ++ return false; ++ } ++ ++ if (!InitReply()) { ++ ERROR("Failed to init reply"); ++ return false; ++ } ++ m_update_req = nullptr; ++ return true; ++} ++ ++auto pluginResult::GetReplyUpdate() -> std::vector ++{ ++ return m_reply.update; ++} ++ ++auto pluginResult::MoveReplyAdjust() -> nri_container_adjustment * ++{ ++ nri_container_adjustment *ret = m_reply.adjust; ++ m_reply.adjust = nullptr; ++ return ret; ++} ++ ++auto pluginResult::GetReplyResources(const std::string &id) -> const nri_linux_resources * ++{ ++ nri_linux_resources *ret = NULL; ++ nri_container_update *update = m_updates[id]; ++ ret = update->linux->resources; ++ return ret; ++} ++ ++auto pluginResult::Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, ++ size_t update_len, const std::string &plugin) -> bool ++{ ++ if (plugin.length() == 0) { ++ ERROR("Empty plugin name"); ++ return false; ++ } ++ if (event == CREATE_CONTAINER) { ++ if (!Adjust(adjust, plugin)) { ++ ERROR("Failed to do adjust to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else if (event == UPDATE_CONTAINER) { ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else if (event == STOP_CONTAINER) { ++ if (!Update(update, update_len, plugin)) { ++ ERROR("Failed to do update to plugin: %s", plugin.c_str()); ++ return false; ++ } ++ return true; ++ } else { ++ ERROR("Cannot apply response of invalid type %d", event); ++ return false; ++ } ++ return true; ++} ++ ++auto pluginResult::Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool ++{ ++ if (adjust == nullptr) { ++ return true; ++ } ++ ++ if (!AdjustAnnotations(adjust->annotations, plugin)) { ++ ERROR("Cannot adajust annotations by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustMounts(adjust->mounts, adjust->mounts_len, plugin)) { ++ ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustEnv(adjust->env, adjust->env_len, plugin)) { ++ ERROR("Cannot adajust mounts by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustHooks(adjust->hooks, plugin)) { ++ ERROR("Cannot adajust hooks by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (adjust->linux != nullptr) { ++ if (m_reply.adjust->linux == nullptr) { ++ m_reply.adjust->linux = (nri_linux_container_adjustment *)util_common_calloc_s(sizeof(nri_linux_container_adjustment)); ++ if (m_reply.adjust->linux == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ if (!AdjustDevices(adjust->linux->devices, adjust->linux->devices_len, plugin)) { ++ ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustResources(adjust->linux->resources, plugin)) { ++ ERROR("Cannot adajust devices by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ if (!AdjustCgroupsPath(adjust->linux->cgroups_path, plugin)) { ++ ERROR("Cannot adajust cgroups path by plugin %s", plugin.c_str()); ++ return false; ++ } ++ } ++ ++ if (!AdjustRlimits(adjust->rlimits, adjust->rlimits_len, plugin)) { ++ ERROR("Cannot adajust rlimits path by plugin %s", plugin.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool ++{ ++ if (annos == nullptr || annos->len == 0) { ++ return true; ++ } ++ ++ if (m_reply.adjust->annotations == nullptr) { ++ m_reply.adjust->annotations = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (m_reply.adjust->annotations == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ google::protobuf::Map del; ++ const char *id = m_conId.c_str(); ++ google::protobuf::Map mapAnno; ++ Transform::JsonMapToProtobufMapForString(annos, mapAnno); ++ ++ // if key is marked for remove, add pair to del, and delete key from annos ++ for (auto it = mapAnno.begin(); it != mapAnno.end();) { ++ const std::string &key = it->first; ++ char *out = NULL; ++ if (is_marked_for_removal(key.c_str(), &out)) { ++ del[out] = ""; ++ it = mapAnno.erase(it); ++ } else { ++ ++it; ++ } ++ } ++ ++ for (const auto &iter : mapAnno) { ++ std::string key = iter.first; ++ std::string value = iter.second; ++ auto it = del.find(key); ++ if (it != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].annotations.erase(key); ++ } ++ append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(key).c_str(), ""); ++ } ++ ++ // set annotations's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto anno = m_owners[id].annotations.find(key); ++ if (anno != m_owners[id].annotations.end()) { ++ ERROR("plugins %s and %s both tried to set annotation: %s", plugin.c_str(), anno->second.c_str(), key.c_str()); ++ return false; ++ } ++ m_owners[id].annotations[key] = plugin; ++ } ++ ++ // add pair to m_reply.adjust ++ append_json_map_string_string(m_reply.adjust->annotations, key.c_str(), value.c_str()); ++ del.erase(key); ++ } ++ ++ // add del to m_reply.adjust ++ for (auto &pair : del) { ++ append_json_map_string_string(m_reply.adjust->annotations, NRIHelpers::MarkForRemoval(pair.first).c_str(), ""); ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool ++{ ++ if (mounts == nullptr || mounts_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector add; ++ std::map del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < mounts_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(mounts[i]->destination, &out)) { ++ del[out] = mounts[i]; ++ } else { ++ add.push_back(mounts[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_mount** cleard = nullptr; ++ size_t clearLen = 0; ++ ++ if (m_reply.adjust->mounts_len > 0) { ++ cleard = (nri_mount **)util_common_calloc_s(m_reply.adjust->mounts_len * sizeof(nri_mount *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < m_reply.adjust->mounts_len; i++) { ++ auto removed = del.find(m_reply.adjust->mounts[i]->destination); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].mounts.erase(m_reply.adjust->mounts[i]->destination); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_mount(m_reply.adjust->mounts[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri mounts to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->mounts, m_reply.adjust->mounts_len); ++ m_reply.adjust->mounts = cleard; ++ m_reply.adjust->mounts_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->mounts_len * sizeof(nri_mount *); ++ newSize = oldSize + add.size() * sizeof(nri_mount *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->mounts), newSize, (void *)m_reply.adjust->mounts, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri mounts array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set mounts's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto mount = m_owners[id].mounts.find(add[i]->destination); ++ if (mount != m_owners[id].mounts.end()) { ++ ERROR("plugins %s and %s both tried to set mount: %s", plugin.c_str(), mount->second.c_str(), add[i]->destination); ++ return false; ++ } ++ m_owners[id].mounts[add[i]->destination] = plugin; ++ } ++ m_reply.adjust->mounts[m_reply.adjust->mounts_len] = copy_nri_mount(add[i]); ++ if (m_reply.adjust->mounts[m_reply.adjust->mounts_len] == nullptr) { ++ ERROR("Failed to copy add nri mounts to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->mounts_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool ++{ ++ if (envs == nullptr || envs_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector add; ++ std::map del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < envs_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(envs[i]->key, &out)) { ++ del[out] = envs[i]; ++ } else { ++ add.push_back(envs[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_key_value** cleard; ++ size_t clearLen = 0; ++ ++ if(m_reply.adjust->env_len > 0) { ++ cleard = (nri_key_value **)util_common_calloc_s(m_reply.adjust->env_len * sizeof(nri_key_value *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < m_reply.adjust->env_len; i++) { ++ auto removed = del.find(m_reply.adjust->env[i]->key); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].env.erase(m_reply.adjust->env[i]->key); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_key_value(m_reply.adjust->env[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri env key value to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->env, m_reply.adjust->env_len); ++ m_reply.adjust->env = cleard; ++ m_reply.adjust->env_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->env_len * sizeof(nri_key_value *); ++ newSize = oldSize + add.size() * sizeof(nri_key_value *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->env), newSize, m_reply.adjust->env, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri env array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set env's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto env = m_owners[id].env.find(add[i]->key); ++ if (env != m_owners[id].env.end()) { ++ ERROR("plugins %s and %s both tried to set env: %s", plugin.c_str(), env->second.c_str(), add[i]->key); ++ return false; ++ } ++ m_owners[id].env[add[i]->key] = plugin; ++ } ++ m_reply.adjust->env[m_reply.adjust->env_len] = copy_nri_key_value(add[i]); ++ if (m_reply.adjust->env[m_reply.adjust->env_len] == nullptr) { ++ ERROR("Failed to copy add nri env to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->env_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool ++{ ++ if (hooks == nullptr) { ++ return true; ++ } ++ ++ if (m_reply.adjust->hooks == nullptr) { ++ m_reply.adjust->hooks = (nri_hooks *)util_common_calloc_s(sizeof(nri_hooks)); ++ if (m_reply.adjust->hooks == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ nri_hooks * reply = m_reply.adjust->hooks; ++ ++ if (!merge_nri_hooks(reply->prestart, reply->prestart_len, (const nri_hook**)hooks->prestart, hooks->prestart_len)) { ++ ERROR("Failed to realloc and copy prestart hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->poststart, reply->poststart_len, (const nri_hook**)hooks->poststart, ++ hooks->poststart_len)) { ++ ERROR("Failed to realloc and copy poststart hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->poststop, reply->poststop_len, (const nri_hook**)hooks->poststop, hooks->poststop_len)) { ++ ERROR("Failed to realloc and copy poststop hooks"); ++ return false; ++ } ++ ++ /* TODO:zhongtao ++ * The OCI being used by the iSulad not supportes ++ * createRuntime/createContainer/startContainer currently. ++ */ ++ if (!merge_nri_hooks(reply->create_runtime, reply->create_runtime_len, (const nri_hook**)hooks->create_runtime, ++ hooks->create_runtime_len)) { ++ ERROR("Failed to realloc and copy create_runtime hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->create_container, reply->create_container_len, (const nri_hook**)hooks->create_container, ++ hooks->create_container_len)) { ++ ERROR("Failed to realloc and copy create_container hooks"); ++ return false; ++ } ++ ++ if (!merge_nri_hooks(reply->start_container, reply->start_container_len, (const nri_hook**)hooks->start_container, ++ hooks->start_container_len)) { ++ ERROR("Failed to realloc and copy start_container hooks"); ++ return false; ++ } ++ ++ return false; ++} ++ ++auto pluginResult::AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool ++{ ++ if (devices_size == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::vector add; ++ std::map del; ++ std::string id = m_conId.c_str(); ++ ++ // first split removals from the rest of adjustments ++ for (i = 0; i < devices_size; i++) { ++ char *out = NULL; ++ if (is_marked_for_removal(devices[i]->path, &out)) { ++ del[out] = devices[i]; ++ } else { ++ add.push_back(devices[i]); ++ } ++ } ++ ++ // next remove marked mounts from collected adjustments ++ nri_linux_device** cleard; ++ size_t clearLen = 0; ++ ++ if (m_reply.adjust->linux->devices_len > 0) { ++ cleard = (nri_linux_device **)util_common_calloc_s(m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *)); ++ if (cleard == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ for (i = 0; i < m_reply.adjust->linux->devices_len; i++) { ++ auto removed = del.find(m_reply.adjust->linux->devices[i]->path); ++ if (removed != del.end()) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ m_owners[id].devices.erase(m_reply.adjust->linux->devices[i]->path); ++ } ++ continue; ++ } ++ cleard[clearLen] = copy_nri_device(m_reply.adjust->linux->devices[i]); ++ if (cleard[clearLen] == nullptr) { ++ ERROR("Failed to copy nri linux device to cleard"); ++ return false; ++ } ++ clearLen++; ++ } ++ ++ NRIHelpers::freeArray(m_reply.adjust->linux->devices, m_reply.adjust->linux->devices_len); ++ m_reply.adjust->linux->devices = cleard; ++ m_reply.adjust->linux->devices_len = clearLen; ++ } ++ ++ // finally, apply additions to collected adjustments ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->linux->devices_len * sizeof(nri_linux_device *); ++ newSize = oldSize + add.size() * sizeof(nri_linux_device *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->linux->devices), newSize, m_reply.adjust->linux->devices, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri devices array"); ++ return false; ++ } ++ for (i = 0; i < add.size(); i++) { ++ // set mounts's owner plugin ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto device = m_owners[id].devices.find(add[i]->path); ++ if (device != m_owners[id].devices.end()) { ++ ERROR("plugins %s and %s both tried to set devices: %s", plugin.c_str(), device->second.c_str(), add[i]->path); ++ return false; ++ } ++ m_owners[id].devices[add[i]->path] = plugin; ++ } ++ m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] = copy_nri_device(add[i]); ++ if (m_reply.adjust->linux->devices[m_reply.adjust->linux->devices_len] == nullptr) { ++ ERROR("Failed to copy add nri devices to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->linux->devices_len++; ++ } ++ ++ return true; ++} ++ ++auto pluginResult::AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool ++{ ++ if (resources == nullptr) { ++ return true; ++ } ++ ++ if (m_reply.adjust->linux->resources == nullptr) { ++ m_reply.adjust->linux->resources = (nri_linux_resources *)util_common_calloc_s(sizeof(nri_linux_resources)); ++ if (m_reply.adjust->linux->resources == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ } ++ ++ std::string id = m_conId.c_str(); ++ nri_linux_resources *reply = m_reply.adjust->linux->resources; ++ ++ return ClaimAndCopyResources(resources, id, plugin, reply); ++} ++ ++bool pluginResult::ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, ++ nri_linux_resources *dest) ++{ ++ size_t i; ++ if (src->memory != nullptr) { ++ if (src->memory->limit != nullptr) { ++ auto memLimit = m_owners[id].memLimit; ++ if (!memLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory limit", plugin.c_str(), memLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memLimit = plugin; ++ *dest->memory->limit = *src->memory->limit; ++ } ++ ++ if (src->memory->reservation != nullptr) { ++ auto memReservation = m_owners[id].memReservation; ++ if (!memReservation.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory reservation", plugin.c_str(), ++ memReservation.c_str()); ++ return false; ++ } ++ m_owners[id].memReservation = plugin; ++ *dest->memory->reservation = *src->memory->reservation; ++ } ++ ++ if (src->memory->swap != nullptr) { ++ auto memSwapLimit = m_owners[id].memSwapLimit; ++ if (!memSwapLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory swap limit", plugin.c_str(), ++ memSwapLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memSwapLimit = plugin; ++ *dest->memory->swap = *src->memory->swap; ++ } ++ ++ if (src->memory->kernel != nullptr) { ++ auto memKernelLimit = m_owners[id].memKernelLimit; ++ if (!memKernelLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory kernel limit", plugin.c_str(), ++ memKernelLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memKernelLimit = plugin; ++ *dest->memory->kernel = *src->memory->kernel; ++ } ++ ++ if (src->memory->kernel_tcp != nullptr) { ++ auto memTCPLimit = m_owners[id].memTCPLimit; ++ if (!memTCPLimit.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory tcp limit", plugin.c_str(), ++ memTCPLimit.c_str()); ++ return false; ++ } ++ m_owners[id].memTCPLimit = plugin; ++ *dest->memory->kernel_tcp = *src->memory->kernel_tcp; ++ } ++ ++ if (src->memory->swappiness != nullptr) { ++ auto memSwappiness = m_owners[id].memSwappiness; ++ if (!memSwappiness.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory swappiness", plugin.c_str(), ++ memSwappiness.c_str()); ++ return false; ++ } ++ m_owners[id].memSwappiness = plugin; ++ *dest->memory->swappiness = *src->memory->swappiness; ++ } ++ ++ if (src->memory->disable_oom_killer != nullptr) { ++ auto memDisableOomKiller = m_owners[id].memDisableOomKiller; ++ if (!memDisableOomKiller.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory disable_oom_killer", plugin.c_str(), ++ memDisableOomKiller.c_str()); ++ return false; ++ } ++ m_owners[id].memDisableOomKiller = plugin; ++ *dest->memory->disable_oom_killer = *src->memory->disable_oom_killer; ++ } ++ ++ if (src->memory->use_hierarchy != nullptr) { ++ auto memUseHierarchy = m_owners[id].memUseHierarchy; ++ if (!memUseHierarchy.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's memory use_hierarchy", plugin.c_str(), ++ memUseHierarchy.c_str()); ++ return false; ++ } ++ m_owners[id].memUseHierarchy = plugin; ++ *dest->memory->use_hierarchy = *src->memory->use_hierarchy; ++ } ++ } ++ ++ if (src->cpu != nullptr) { ++ if (src->cpu->shares != nullptr) { ++ auto cpuShares = m_owners[id].cpuShares; ++ if (!cpuShares.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu shares", plugin.c_str(), cpuShares.c_str()); ++ return false; ++ } ++ m_owners[id].cpuShares = plugin; ++ *dest->cpu->shares = *src->cpu->shares; ++ } ++ ++ if (src->cpu->quota != nullptr) { ++ auto cpuQuota = m_owners[id].cpuQuota; ++ if (!cpuQuota.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu quota", plugin.c_str(), cpuQuota.c_str()); ++ return false; ++ } ++ m_owners[id].cpuQuota = plugin; ++ *dest->cpu->quota = *src->cpu->quota; ++ } ++ ++ if (src->cpu->period != nullptr) { ++ auto cpuPeriod = m_owners[id].cpuPeriod; ++ if (!cpuPeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu period", plugin.c_str(), cpuPeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuPeriod = plugin; ++ *dest->cpu->period = *src->cpu->period; ++ } ++ ++ if (src->cpu->realtime_runtime != nullptr) { ++ auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; ++ if (!cpuRealtimePeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu realtime_runtime", plugin.c_str(), ++ cpuRealtimePeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuRealtimePeriod = plugin; ++ *dest->cpu->realtime_runtime = *src->cpu->realtime_runtime; ++ } ++ ++ if (src->cpu->realtime_period != nullptr) { ++ auto cpuRealtimePeriod = m_owners[id].cpuRealtimePeriod; ++ if (!cpuRealtimePeriod.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu realtime_period", plugin.c_str(), ++ cpuRealtimePeriod.c_str()); ++ return false; ++ } ++ m_owners[id].cpuRealtimePeriod = plugin; ++ *dest->cpu->realtime_period = *src->cpu->realtime_period; ++ } ++ ++ if (src->cpu->cpus != nullptr) { ++ auto cpusetCpus = m_owners[id].cpusetCpus; ++ if (!cpusetCpus.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu cpus", plugin.c_str(), cpusetCpus.c_str()); ++ return false; ++ } ++ m_owners[id].cpusetCpus = plugin; ++ *dest->cpu->cpus = *src->cpu->cpus; ++ } ++ ++ if (src->cpu->mems != nullptr) { ++ auto cpusetMems = m_owners[id].cpusetMems; ++ if (!cpusetMems.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cpu mems", plugin.c_str(), cpusetMems.c_str()); ++ return false; ++ } ++ m_owners[id].cpusetMems = plugin; ++ *dest->cpu->mems = *src->cpu->mems; ++ } ++ } ++ ++ for (i = 0; i < src->hugepage_limits_len; i++) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto find = m_owners[id].hugepageLimits.find(src->hugepage_limits[i]->page_size); ++ if (find != m_owners[id].hugepageLimits.end()) { ++ ERROR("plugins %s and %s both tried to set hugepageLimits: %s", plugin.c_str(), find->second.c_str(), ++ src->hugepage_limits[i]->page_size); ++ return false; ++ } ++ m_owners[id].hugepageLimits[src->hugepage_limits[i]->page_size] = plugin; ++ } ++ } ++ ++ if (src->unified->len != 0) { ++ google::protobuf::Map mapAnno; ++ Transform::JsonMapToProtobufMapForString(src->unified, mapAnno); ++ for (const auto &iter : mapAnno) { ++ std::string key = iter.first; ++ std::string value = iter.second; ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto anno = m_owners[id].unified.find(key); ++ if (anno != m_owners[id].unified.end()) { ++ ERROR("plugins %s and %s both tried to set unified: %s", plugin.c_str(), anno->second.c_str(), ++ key.c_str()); ++ return false; ++ } ++ m_owners[id].unified[key] = plugin; ++ } ++ // add pair to m_reply.adjust ++ append_json_map_string_string(dest->unified, key.c_str(), value.c_str()); ++ } ++ } ++ ++ if (src->blockio_class != nullptr) { ++ auto blockioClass = m_owners[id].blockioClass; ++ if (!blockioClass.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's blockio_class", plugin.c_str(), blockioClass.c_str()); ++ return false; ++ } ++ m_owners[id].blockioClass = plugin; ++ dest->blockio_class = util_strdup_s(src->blockio_class); ++ } ++ ++ if (src->rdt_class != nullptr) { ++ auto rdtClass = m_owners[id].rdtClass; ++ if (!rdtClass.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's rdt_class", plugin.c_str(), rdtClass.c_str()); ++ return false; ++ } ++ m_owners[id].rdtClass = plugin; ++ dest->rdt_class = util_strdup_s(src->rdt_class); ++ } ++ return true; ++} ++ ++auto pluginResult::AdjustCgroupsPath(char *path, const std::string &plugin) -> bool ++{ ++ if (path == nullptr || strcmp(path, "") == 0) { ++ return true; ++ } ++ ++ std::string id = m_conId.c_str(); ++ ++ auto cgroupsPath = m_owners[id].cgroupsPath; ++ if (!cgroupsPath.empty()) { ++ ERROR("plugins %s and %s both tried to set devices's cgroups path", plugin.c_str(), cgroupsPath.c_str()); ++ return false; ++ } ++ m_owners[id].cgroupsPath = plugin; ++ m_reply.adjust->linux->cgroups_path = util_strdup_s(path); ++ ++ return true; ++} ++ ++auto pluginResult::AdjustRlimits(nri_posix_rlimit **rlimits, size_t rlimits_len, const std::string &plugin) -> bool ++{ ++ if (rlimits_len == 0) { ++ return true; ++ } ++ ++ size_t i; ++ std::string id = m_conId.c_str(); ++ ++ size_t oldSize, newSize; ++ oldSize = m_reply.adjust->rlimits_len * sizeof(nri_posix_rlimit *); ++ newSize = oldSize + rlimits_len * sizeof(nri_posix_rlimit *); ++ int ret = util_mem_realloc((void **)(&m_reply.adjust->rlimits), newSize, m_reply.adjust->rlimits, oldSize); ++ if (ret != 0) { ++ ERROR("Failed to realloc and assign nri rlimits array"); ++ return false; ++ } ++ ++ for (i = 0; i < rlimits_len; i++) { ++ auto owner = m_owners.find(id); ++ if (owner != m_owners.end()) { ++ auto find = m_owners[id].rlimits.find(rlimits[i]->type); ++ if (find != m_owners[id].rlimits.end()) { ++ ERROR("plugins %s and %s both tried to set rlimits type: %s", plugin.c_str(), find->second.c_str(), rlimits[i]->type); ++ return false; ++ } ++ m_owners[id].rlimits[rlimits[i]->type] = plugin; ++ } ++ m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] = copy_nri_posix_rlimit(rlimits[i]); ++ if (m_reply.adjust->rlimits[m_reply.adjust->rlimits_len] == nullptr) { ++ ERROR("Failed to copy add nri rlimits to reply adjust"); ++ return false; ++ } ++ m_reply.adjust->rlimits_len++; ++ } ++ return true; ++} ++ ++auto pluginResult::Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool ++{ ++ if (update_len == 0) { ++ return true; ++ } ++ ++ size_t i; ++ ++ for (i = 0; i < update_len; i++) { ++ nri_container_update *reply; ++ if (!GetContainerUpdate(updates[i], plugin, &reply)) { ++ ERROR("Failed to get container update in plugin result"); ++ return false; ++ } ++ ++ if (!UpdateResources(reply, updates[i], plugin)) { ++ ERROR("Failed to update container resources in plugin result"); ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++auto pluginResult::GetContainerUpdate(nri_container_update *update, const std::string &plugin, ++ nri_container_update **out) -> bool ++{ ++ if (update == nullptr || out == nullptr || plugin.empty()) { ++ ERROR("Empyt input args"); ++ return false; ++ } ++ ++ auto id = update->container_id; ++ ++ if (std::string(id) == m_conId) { ++ ERROR("Plugin %s asked update of %s during creation", plugin.c_str(), id); ++ return false; ++ } ++ ++ auto find = m_updates.find(id); ++ if (find != m_updates.end()) { ++ *out = m_updates[id]; ++ (*out)->ignore_failure = (*out)->ignore_failure && update->ignore_failure; ++ return true; ++ } ++ ++ *out = init_nri_container_update(id, update->ignore_failure); ++ if (*out == nullptr) { ++ ERROR("Failed to init nri container update"); ++ return false; ++ } ++ ++ m_updates[id] = *out; ++ ++ // for update requests delay appending the requested container (in the response getter) ++ if (m_conId != id) { ++ m_reply.update.push_back(*out); ++ } ++ ++ return true; ++} ++ ++auto pluginResult::UpdateResources(nri_container_update *reply, nri_container_update *u, ++ const std::string &plugin) -> bool ++{ ++ if (u->linux == nullptr || u->linux->resources == nullptr) { ++ return true; ++ } ++ ++ std::string id = u->container_id; ++ nri_linux_resources *resources; ++ ++ if (m_conId == id) { ++ resources = copy_nri_linux_resources(m_update_req); ++ if (resources == nullptr) { ++ ERROR("Failed to copy request's nri linux resources"); ++ return false; ++ } ++ } else { ++ resources = copy_nri_linux_resources(reply->linux->resources); ++ if (resources == nullptr) { ++ ERROR("Failed to copy reply's nri linux resources"); ++ return false; ++ } ++ } ++ ++ if (!ClaimAndCopyResources(u->linux->resources, id, plugin, resources)) { ++ ERROR("Failed to claim and copy resources in plugin result"); ++ return false; ++ } ++ ++ // update reply from copy on success ++ reply->linux->resources = copy_nri_linux_resources(resources); ++ if (reply->linux->resources == nullptr) { ++ ERROR("Failed to copy resources's nri linux resources to reply"); ++ return false; ++ } ++ ++ return true; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/nri_result.h b/src/daemon/nri/nri_result.h +index f2896ea0..4f26385a 100644 +--- a/src/daemon/nri/nri_result.h ++++ b/src/daemon/nri/nri_result.h +@@ -63,7 +63,7 @@ struct owners { + + struct resultReply { + nri_container_adjustment* adjust; +- std::vector update; ++ std::vector update; + }; + + using resultOwners = std::map; +@@ -71,16 +71,19 @@ using resultOwners = std::map; + class pluginResult { + public: + pluginResult() = default; ++ pluginResult(std::string conId); + +- ~pluginResult() = default; ++ virtual ~pluginResult(); + ++ auto Init() -> bool; + auto InitByConId(std::string conId) -> bool; + auto InitByUpdateReq(nri_update_container_request *req) -> bool; + + auto GetReplyUpdate() -> std::vector; +- auto GetReplyAdjust() -> nri_container_adjustment *; ++ auto MoveReplyAdjust() -> nri_container_adjustment *; ++ auto GetReplyResources(const std::string &id) -> const nri_linux_resources *; + +- auto Apply(int32_t event, nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, ++ auto Apply(int32_t event, const nri_container_adjustment *adjust, nri_container_update **update, size_t update_len, + const std::string &plugin) -> bool; + auto Update(nri_container_update **updates, size_t update_len, const std::string &plugin) -> bool; + +@@ -90,12 +93,12 @@ private: + + auto InitReply(void) -> bool; + +- auto Adjust(nri_container_adjustment *adjust, const std::string &plugin) -> bool; ++ auto Adjust(const nri_container_adjustment *adjust, const std::string &plugin) -> bool; + + auto AdjustAnnotations(json_map_string_string *annos, const std::string &plugin) -> bool; + auto AdjustMounts(nri_mount **mounts, size_t mounts_size, const std::string &plugin) -> bool; + auto AdjustEnv(nri_key_value **envs, size_t envs_size, const std::string &plugin) -> bool; +- auto AdjustHooks(nri_hooks *hooks, const std::string &plugin) -> bool; ++ auto AdjustHooks(const nri_hooks *hooks, const std::string &plugin) -> bool; + auto AdjustDevices(nri_linux_device **devices, size_t devices_size, const std::string &plugin) -> bool; + auto AdjustResources(nri_linux_resources *resources, const std::string &plugin) -> bool; + bool ClaimAndCopyResources(nri_linux_resources *src, std::string &id, const std::string &plugin, +@@ -107,7 +110,9 @@ private: + std::string m_conId; + nri_linux_resources *m_update_req; + resultReply m_reply; ++ // update plugin -> update context + std::map m_updates; ++ // adjust plugin -> adjust context + resultOwners m_owners; + }; + +diff --git a/src/daemon/nri/plugin.cc b/src/daemon/nri/plugin.cc +new file mode 100644 +index 00000000..904677c4 +--- /dev/null ++++ b/src/daemon/nri/plugin.cc +@@ -0,0 +1,417 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: zhongtao ++ * Create: 2024-03-15 ++ * Description: provide plugin class definition ++ *********************************************************************************/ ++ ++#include "plugin.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "cstruct_wrapper.h" ++ ++// same as containerd ++std::string DefaultNRIVersion = "2.0.0-beta.2+unknown"; ++// same as containerd ++std::string DefaultNRIRuntimeName = "v2"; ++// defualt timeout for wait: 2s ++const int64_t DefaultWaitTimeout = 2000; ++const uint64_t SECOND_TO_NANOS = 1000000000; ++ ++// init client conn ++NRIPlugin::NRIPlugin(std::string &idx, std::string &name, std::string &config) ++{ ++ m_idx = idx; ++ m_name = name; ++ m_config = config; ++ m_closed = false; ++ m_external = false; ++ m_pid = -1; ++} ++ ++NRIPlugin::NRIPlugin(int fd, std::string &name) ++{ ++ m_sockFds.push_back(fd); ++ m_name = name; ++ m_closed = false; ++ m_external = true; ++ m_pid = -1; ++} ++ ++// wait for plugin to register, then configure it. ++auto NRIPlugin::Start(int64_t registry_timeout, int64_t request_timeout) -> bool ++{ ++ Errors error; ++ ++ // todo: what if timeout is 0 or other invalid value? ++ ++ if (!Connect(request_timeout)) { ++ ERROR("Failed to connect nri plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ if (!WaitForReady(registry_timeout)) { ++ ERROR("Failed to wait plugin %s ready with timeout %ld", m_name.c_str(), registry_timeout); ++ return false; ++ } ++ ++ if (!Configure(error)) { ++ ERROR("Failed to configure nri plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::shutdown() -> void ++{ ++ if (!Close()) { ++ ERROR("Failed to close plugin %s", m_name.c_str()); ++ } ++ ++ if (!Stop()) { ++ ERROR("Failed to stop plugin %s", m_name.c_str()); ++ } ++} ++ ++// create client connect ++auto NRIPlugin::Connect(int64_t timeout) -> bool ++{ ++ if (m_name.empty()) { ++ ERROR("Empty nri plugin name"); ++ return false; ++ } ++ ++ if (nri_plugin_connect(m_name.c_str(), m_sockFds[0], timeout * SECOND_TO_NANOS) != 0) { ++ ERROR("Failed to create a new client for plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++// close a plugin shutting down its multiplexed ttrpc connections. ++auto NRIPlugin::Close() -> bool ++{ ++ if (IsClose()) { ++ return true; ++ } ++ ++ if (nri_plugin_disconnect(m_name.c_str()) != 0) { ++ ERROR("Failed to close plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ SetClose(); ++ return true; ++} ++ ++// stop a plugin (if it was launched by us) ++auto NRIPlugin::Stop() -> bool ++{ ++ if (m_external) { ++ return true; ++ } ++ ++ if (m_pid <= 0) { ++ WARN("Invalid pid %d", m_pid); ++ return false; ++ } ++ ++ int nret = kill(m_pid, SIGKILL); ++ if (nret < 0 && errno != ESRCH) { ++ SYSWARN("Can not kill process (pid=%d) with SIGKILL", m_pid); ++ return false; ++ } ++ ++ if (util_waitpid_with_timeout(m_pid, DefaultWaitTimeout, NULL) != 0) { ++ WARN("Failed to wait for plugin %s to exit", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++// Name returns a string indentication for the plugin. ++auto NRIPlugin::GetName() -> const std::string & ++{ ++ return m_name; ++} ++ ++auto NRIPlugin::GetIndex() -> const std::string & ++{ ++ return m_idx; ++} ++ ++auto NRIPlugin::GetPeerSockFd() -> uint32_t ++{ ++ return m_sockFds[1]; ++} ++ ++auto NRIPlugin::GetQualifiedName() -> std::string ++{ ++ return m_idx + "-" + m_name; ++} ++ ++void NRIPlugin::SetReady(void) ++{ ++ std::unique_lock lock(m_readyMutex); ++ m_ready = true; ++ m_condition.notify_one(); ++} ++ ++void NRIPlugin::SetPid(int pid) ++{ ++ m_pid = pid; ++} ++ ++auto NRIPlugin::CreateSocketPair() -> bool ++{ ++ int fds[2]; ++ ++ if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) { ++ ERROR("Failed to create socketpair"); ++ return false; ++ } ++ ++ m_sockFds.push_back(fds[0]); ++ m_sockFds.push_back(fds[1]); ++ return true; ++} ++ ++auto NRIPlugin::Configure(Errors &error) -> bool ++{ ++ auto req = makeUniquePtrCStructWrapper(free_nri_configure_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ req->get()->config = isula_strdup_s(m_config.c_str()); ++ req->get()->runtime_name = isula_strdup_s(NRIRruntime.c_str()); ++ req->get()->runtime_version = isula_strdup_s(NRIVersion.c_str()); ++ ++ nri_configure_response *resp = nullptr; ++ if (nri_plugin_configure(m_name.c_str(), req->get(), &resp) != 0) { ++ ERROR("Failed to configure plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_configure_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ EventMask events = resp_wrapper->get()->events; ++ if (events != 0) { ++ EventMask extra = events & ~ValidEvents; ++ if (extra != 0) { ++ ERROR("Invalid plugin events: %d", extra); ++ return false; ++ } ++ } else { ++ events = ValidEvents; ++ } ++ ++ m_events = events; ++ return true; ++} ++ ++auto NRIPlugin::Synchronize(std::vector &pods, std::vector &containers, ++ nri_container_update ***update, size_t update_len, Errors &error) -> bool ++{ ++ size_t i; ++ ++ auto req = makeUniquePtrCStructWrapper(free_nri_synchronize_request); ++ if (req == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ if (pods.size() != 0) { ++ req->get()->pods = (nri_pod_sandbox **)util_common_calloc_s(pods.size() * sizeof(nri_pod_sandbox *)); ++ if (req->get()->pods == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < pods.size(); i++) { ++ req->get()->pods[i] = pods[i]; ++ req->get()->pods_len++; ++ } ++ } ++ ++ if (containers.size() != 0) { ++ req->get()->containers = (nri_container **)util_common_calloc_s(containers.size() * sizeof(nri_container *)); ++ if (req->get()->containers == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ for (i = 0; i < containers.size(); i++) { ++ req->get()->containers[i] = containers[i]; ++ req->get()->containers_len++; ++ } ++ } ++ ++ nri_synchronize_response *resp = nullptr; ++ if (nri_plugin_synchronize(m_name.c_str(), req->get(), &resp) != 0) { ++ ERROR("Failed to synchronize plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ auto resp_wrapper = makeUniquePtrCStructWrapper(resp, free_nri_synchronize_response); ++ if (resp_wrapper == nullptr) { ++ ERROR("Out of memory"); ++ return false; ++ } ++ ++ *update = resp->update; ++ resp->update = nullptr; ++ update_len = resp->update_len; ++ resp->update_len = 0; ++ return true; ++} ++ ++auto NRIPlugin::CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (IsSetEvent(CREATE_CONTAINER) == false) { ++ return true; ++ } ++ ++ if (nri_plugin_create_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to create container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(UPDATE_CONTAINER)) { ++ return true; ++ } ++ ++ if (nri_plugin_update_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to update container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIPlugin::StopContainer(nri_stop_container_request *req, nri_stop_container_response **resp, ++ Errors &error) -> bool ++{ ++ if (req == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(STOP_CONTAINER)) { ++ return true; ++ } ++ ++ if (nri_plugin_stop_container(m_name.c_str(), req, resp) != 0) { ++ ERROR("Failed to stop container by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++// do nothing with event ++auto NRIPlugin::StateChange(nri_state_change_event *evt, Errors &error) -> bool ++{ ++ if (evt == nullptr) { ++ ERROR("Invalid input"); ++ return false; ++ } ++ ++ if (!IsSetEvent(evt->event)) { ++ return true; ++ } ++ ++ if (nri_plugin_state_change(m_name.c_str(), evt) != 0) { ++ ERROR("Failed to state change by plugin %s", m_name.c_str()); ++ return false; ++ } ++ return true; ++} ++ ++auto NRIPlugin::WaitForReady(int64_t timeout) -> bool ++{ ++ auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout * 1000); ++ std::unique_lock readyMutex(m_readyMutex); ++ ++ if (timeout == 0) { ++ m_condition.wait(readyMutex); ++ return true; ++ } ++ ++ if (m_condition.wait_until(readyMutex, deadline) == std::cv_status::timeout) { ++ return false; ++ } ++ ++ return true; ++} ++ ++auto NRIPlugin::IsSetEvent(EventMask e) -> bool ++{ ++ return (m_events & (1 << (e - 1))) != 0; ++} ++ ++auto NRIPlugin::IsClose() -> bool ++{ ++ ReadGuard lock(m_mutex); ++ return m_closed; ++} ++ ++void NRIPlugin::SetClose() ++{ ++ WriteGuard lock(m_mutex); ++ m_closed = true; ++} +\ No newline at end of file +diff --git a/src/daemon/nri/plugin.h b/src/daemon/nri/plugin.h +index f60a9b3d..ed298be6 100644 +--- a/src/daemon/nri/plugin.h ++++ b/src/daemon/nri/plugin.h +@@ -38,17 +38,14 @@ public: + virtual ~NRIPlugin() = default; + // wait for plugin to register, then configure it. + auto Start(int64_t registry_timeout, int64_t request_timeout) -> bool; +- // close a plugin shutting down its multiplexed ttrpc connections. +- auto Close(void) -> bool; +- // stop a plugin (if it was launched by us) +- auto Stop(void) -> bool; ++ ++ auto shutdown() -> void; + + // Name returns a string indentication for the plugin. + auto GetName(void) -> const std::string &; + auto GetIndex(void) -> const std::string &; + auto GetPeerSockFd(void) -> uint32_t; + auto GetQualifiedName(void) -> std::string; +- + void SetReady(void); + void SetPid(int pid); + +@@ -58,8 +55,8 @@ public: + + auto Configure(Errors &error) -> bool; + // Only called in external plugin scenario +- auto Synchronize(std::vector> pods, +- std::vector> &containers, nri_container_update ***update, size_t update_len, ++ auto Synchronize(std::vector &pods, ++ std::vector &containers, nri_container_update ***update, size_t update_len, + Errors &error) -> bool; + auto CreateContainer(nri_create_container_request *req, nri_create_container_response **resp, Errors &error) -> bool; + auto UpdateContainer(nri_update_container_request *req, nri_update_container_response **resp, Errors &error) -> bool; +@@ -72,6 +69,13 @@ private: + auto WaitForReady(int64_t timeout) -> bool; + auto IsSetEvent(EventMask e) -> bool; + ++ // close a plugin shutting down its multiplexed ttrpc connections. ++ auto Close(void) -> bool; ++ // stop a plugin (if it was launched by us) ++ auto Stop(void) -> bool; ++ ++ void SetClose(void); ++ + private: + RWMutex m_mutex; + bool m_external; +@@ -83,7 +87,7 @@ private: + std::vector m_sockFds; + std::string m_localFileName; + std::string m_peerFileName; +- // TODO:zhontao monitor? ++ // TODO: plugin monitor? + bool m_closed; + std::mutex m_readyMutex; + bool m_ready; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index f81a9141..69f6dbf0 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1705,4 +1705,4 @@ void set_child_process_pdeathsig(void) + if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) { + SYSERROR("Failed to set child process pdeathsig"); + } +-} +\ No newline at end of file ++} +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec index 9c52cb6..6db1f73 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.1.5 -%global _release 10 +%global _release 11 %global is_systemd 1 %global enable_criv1 1 %global enable_cdi 1 @@ -124,6 +124,19 @@ Patch0105: 0105-start-sandbox-before-setup-network-by-default.patch Patch0106: 0106-Revert-use-isula_clean_path-rather-than-realpath.patch Patch0107: 0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch Patch0108: 0108-skip-test-rely-on-docker.io.patch +Patch0109: 0109-modify-default-registry-mirrors-in-ci-test.patch +Patch0110: 0110-add-timestamp-in-PodSandboxStatu-response.patch +Patch0111: 0111-bugfix-for-file-param-verify.patch +Patch0112: 0112-bugfix-change-cni-log-info.patch +Patch0113: 0113-move-shutdown-handle-after-init-module.patch +Patch0114: 0114-bugfix-for-null-pointer-reference.patch +Patch0115: 0115-bugfix-for-m_criService-shutdown.patch +Patch0116: 0116-fix-bug-in-ci-test.patch +Patch0117: 0117-NRI-add-nri-head-file-and-common-func.patch +Patch0118: 0118-skip-calling-cni-plugin-cleanup-when-network-namespa.patch +Patch0119: 0119-nri-add-convert-and-utils-impl-for-nri.patch +Patch0120: 0120-get-realpath-before-ns-mountpoint-verification.patch +Patch0121: 0121-nri-impl-for-nri-plugin-and-adaption.patch %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) @@ -380,6 +393,12 @@ fi %endif %changelog +* Mon Aug 19 2024 zhongtao - 2.1.5-11 +- Type: update +- ID: NA +- SUG: NA +- DESC: add impl for nri and bugfix + * Tue Jun 11 2024 zhongtao - 2.1.5-10 - Type: update - ID: NA -- Gitee